\PassOptionsToPackage{unicode=true}{hyperref} % options for packages loaded elsewhere
\PassOptionsToPackage{hyphens}{url}
\PassOptionsToPackage{dvipsnames,svgnames*,x11names*}{xcolor}
%
\documentclass[11pt,fleqn]{article}
\usepackage{lmodern}
\usepackage{amssymb,amsmath}
\usepackage{ifxetex,ifluatex}
\usepackage{fixltx2e} % provides \textsubscript
\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
  \usepackage[T1]{fontenc}
  \usepackage[utf8]{inputenc}
  \usepackage{textcomp} % provides euro and other symbols
\else % if luatex or xelatex
  \usepackage{unicode-math}
  \defaultfontfeatures{Ligatures=TeX,Scale=MatchLowercase}
\fi
% use upquote if available, for straight quotes in verbatim environments
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
% use microtype if available
\IfFileExists{microtype.sty}{%
\usepackage[]{microtype}
\UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
}{}
\IfFileExists{parskip.sty}{%
\usepackage{parskip}
}{% else
\setlength{\parindent}{0pt}
\setlength{\parskip}{6pt plus 2pt minus 1pt}
}
\usepackage{xcolor}
\usepackage{hyperref}
\hypersetup{
            pdftitle={ATC: an Advanced Tucker-based Compression library for multidimensional data -- User manual},
            pdfauthor={Wouter Baert},
            colorlinks=true,
            linkcolor=blue,
            filecolor=Maroon,
            citecolor=Blue,
            urlcolor=blue,
            breaklinks=true}
\urlstyle{same}  % don't use monospace font for urls
\usepackage[margin=2cm]{geometry}
\usepackage{longtable,booktabs}
% Fix footnotes in tables (requires footnote package)
\IfFileExists{footnote.sty}{\usepackage{footnote}\makesavenoteenv{longtable}}{}
\usepackage{graphicx,grffile}
\makeatletter
\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
\makeatother
% Scale images if necessary, so that they will not overflow the page
% margins by default, and it is still possible to overwrite the defaults
% using explicit options in \includegraphics[width, height, ...]{}
\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
\setlength{\emergencystretch}{3em}  % prevent overfull lines
\providecommand{\tightlist}{%
  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
\setcounter{secnumdepth}{0}
% Redefines (sub)paragraphs to behave more like sections
\ifx\paragraph\undefined\else
\let\oldparagraph\paragraph
\renewcommand{\paragraph}[1]{\oldparagraph{#1}\mbox{}}
\fi
\ifx\subparagraph\undefined\else
\let\oldsubparagraph\subparagraph
\renewcommand{\subparagraph}[1]{\oldsubparagraph{#1}\mbox{}}
\fi

% set default figure placement to htbp
\makeatletter
\def\fps@figure{htbp}
\makeatother

\usepackage{fvextra}
\usepackage[htt]{hyphenat}
\let\NormalTok\texttt
\DefineVerbatimEnvironment{Highlighting}{Verbatim}{breaklines, breakanywhere, breaksymbol=, breakanywheresymbolpre=, commandchars=\\\{\}}

\title{ATC: an Advanced Tucker-based Compression library for multidimensional
data -- User manual}
\providecommand{\subtitle}[1]{}
\subtitle{User manual}
\author{Wouter Baert}
\date{}

\begin{document}
\maketitle

ATC is a compression library for numerical data on grids of 3 dimensions
or higher, aimed at optimizing compression rates,
compression/decompression time, memory usage and error control. It
mainly relies on the Tucker decomposition, which exploits correlations
along each mode (dimension) but takes some time to compute. Furthermore,
ATC controls the error of the resulting tensor in terms of the
Euclidean/Frobenius norm, in contrast to some other compressors which
enforce an elementwise error bound or use other error metrics.

For an elaborate description of the algorithms used in ATC, as well as
various implementation aspects, please consult the
\href{https://doi.org/10.48550/arxiv.2107.01384}{our paper}.

ATC was developed by Wouter Baert as part of a research project under
supervision of Nick Vannieuwenhoven at KU Leuven, Belgium.

\hypertarget{main-features}{%
\subsection{Main features}\label{main-features}}

\begin{itemize}
\tightlist
\item
  Interfaces for C++ (using STL containers), C and command-line
\item
  Error control in terms of the Frobenius norm of the error tensor
  (represented as relative error, RMSE or PSNR)
\item
  Multi-threading support for linear algebra, DCT's, tensor
  transpositions as well as quantization and encoding
\item
  Support for efficient decompression of specific subsets of the
  original data, using cut-outs and optionally downsampling
\item
  Support for compression/decompression to/from files (C++/C/CLI) or
  in-memory (only C++/C)
\item
  Support for 10 input data types (\texttt{uint8}, \texttt{uint16},
  \texttt{uint32}, \texttt{uint64}, \texttt{int8}, \texttt{int16},
  \texttt{int32}, \texttt{int64}, \texttt{float32}, \texttt{float64})
\item
  Support for single and double precision for floating-point arithmetic
\end{itemize}

\hypertarget{main-algorithmic-components}{%
\subsection{Main algorithmic
components}\label{main-algorithmic-components}}

\begin{itemize}
\tightlist
\item
  ST-HOSVD-based truncated Tucker decomposition
\item
  Parallellized quantization and encoding based on split bit plane
  truncation
\item
  Precise error control using dequantization correction prediction
\end{itemize}

\newpage

\hypertarget{compression-examples}{%
\subsection{Compression examples}\label{compression-examples}}

\small

\begin{longtable}[]{@{}cccc@{}}
\toprule
\begin{minipage}[b]{0.22\columnwidth}\centering
\includegraphics{../../images/compression-example_Isotropic1024-fine-pressure-512_original.png}\strut
\end{minipage} & \begin{minipage}[b]{0.22\columnwidth}\centering
\includegraphics{../../images/compression-example_Isotropic1024-fine-pressure-512_0-010.png}\strut
\end{minipage} & \begin{minipage}[b]{0.22\columnwidth}\centering
\includegraphics{../../images/compression-example_Isotropic1024-fine-pressure-512_0-030.png}\strut
\end{minipage} & \begin{minipage}[b]{0.22\columnwidth}\centering
\includegraphics{../../images/compression-example_Isotropic1024-fine-pressure-512_0-100.png}\strut
\end{minipage}\tabularnewline
\midrule
\endhead
\begin{minipage}[t]{0.22\columnwidth}\centering
Original\strut
\end{minipage} & \begin{minipage}[t]{0.22\columnwidth}\centering
Compression factor: 153\strut
\end{minipage} & \begin{minipage}[t]{0.22\columnwidth}\centering
Compression factor: 639\strut
\end{minipage} & \begin{minipage}[t]{0.22\columnwidth}\centering
Compression factor: 4948\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.22\columnwidth}\centering
\strut
\end{minipage} & \begin{minipage}[t]{0.22\columnwidth}\centering
Relative error: 1.00\%\strut
\end{minipage} & \begin{minipage}[t]{0.22\columnwidth}\centering
Relative error: 3.00\%\strut
\end{minipage} & \begin{minipage}[t]{0.22\columnwidth}\centering
Relative error: 10.1\%\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.22\columnwidth}\centering
\strut
\end{minipage} & \begin{minipage}[t]{0.22\columnwidth}\centering
PSNR: 63.9 dB\strut
\end{minipage} & \begin{minipage}[t]{0.22\columnwidth}\centering
PSNR: 54.3 dB\strut
\end{minipage} & \begin{minipage}[t]{0.22\columnwidth}\centering
PSNR: 43.7 dB\strut
\end{minipage}\tabularnewline
\bottomrule
\end{longtable}

\normalsize

Dataset: A 1x512x512x512 cut-out, selecting the subtensor with the
lowest coordinates, of the ``Isotropic 1024 Fine'' dataset from the
\href{http://turbulence.pha.jhu.edu}{Johns Hopkins Turbulence Database}
{[}1{]}.

\hypertarget{usage}{%
\subsection{Usage}\label{usage}}

\hypertarget{requirements}{%
\subsubsection{Requirements}\label{requirements}}

ATC is written in C++ using features from C++17 and lower, so first of
all a compatible compiler is required (e.g.~GCC version 7 or higher).
\href{https://cmake.org/}{CMake} is used to generate the Make build
system. Furthermore, the following software must be installed on the
system in a place where CMake will find them:

\begin{itemize}
\tightlist
\item
  \href{https://eigen.tuxfamily.org}{Eigen}
\item
  Optional, to enable \href{http://www.netlib.org/blas/}{BLAS} {[}2{]}
  routines: a \href{http://www.netlib.org/blas/}{BLAS} implementation
  (only used when using \texttt{-DATC\_BLAS=ON}, more information below)
\item
  Optional, to enable multi-threading:
  \href{https://www.openmp.org/}{OpenMP} {[}3{]}
\item
  Optional, to compile the command-line interface executable:
  \href{https://www.boost.org/doc/libs/1_75_0/doc/html/program_options.html}{Boost.Program\_Options}
\item
  Optional, to enable T-HODCT preprocessing:
  \href{http://www.fftw.org/}{FFTW} {[}4{]}
\end{itemize}

The following libraries are also used in ATC but are included already
and do not require any further set-up from the user:

\begin{itemize}
\tightlist
\item
  \href{https://github.com/springer13/hptt}{HPTT} {[}5{]}
\item
  \href{https://github.com/rballester/tthresh}{Parts of TTHRESH} {[}6{]}
  (with permission from the author)
\end{itemize}

\hypertarget{installation}{%
\subsubsection{Installation}\label{installation}}

You can perform an out-of-source build by executing the following
commands in the root directory:

\begin{verbatim}
mkdir build
cd build
cmake ..
make cli -j4
\end{verbatim}

This will build the library itself as well as the command-line interface
executable. If one of the aforementioned libraries is installed on your
system, but the CMake configuration process does not find them (as shown
in the CMake output), you can add a macro definition to the
\texttt{cmake\ ..} command in order to point CMake towards the
installation directory. Specifically, it should point to a directory
containing the relevant CMake files, such as
\texttt{\textless{}package\textgreater{}Config.cmake} or
\texttt{\textless{}package\textgreater{}-config.cmake}. For example, if
Eigen is not found, you might need to add
\texttt{-DEigen3\_DIR=\textless{}path\textgreater{}/eigen3/cmake/},
because this is where \texttt{Eigen3Config.cmake} is located. Note that
for the optional libraries, if CMake doesn't find them, it will not
print a message as to how to help it locate them. For detailed
information, see the
\href{https://cmake.org/cmake/help/latest/command/find_package.html}{relevant
CMake documentation}.

Furthermore, if you want ATC to use BLAS, you should add the macro
definition \texttt{-DATC\_BLAS=ON} to the \texttt{cmake\ ..} command. In
this case, CMake will automatically select a BLAS implementation found
on the system, or report an error if none are found. If you want to
ensure that a certain vendor is used, you can specify this using
\texttt{-DBLA\_VENDOR=\textless{}vendor\textgreater{}}, where
\texttt{\textless{}vendor\textgreater{}} can be \texttt{OpenBLAS},
\texttt{Intel10\_64lp}, \texttt{Goto}, \ldots{} For more information,
see the
\href{https://cmake.org/cmake/help/latest/module/FindBLAS.html}{relevant
CMake documentation}.

Finally, if you want to install ATC globally on your machine (or at the
user level), you may want to add the flag
\texttt{-DCMAKE\_INSTALL\_PREFIX=\textless{}path\textgreater{}} to the
\texttt{cmake\ ..} command. For example, on Unix systems, CMake will set
this variable to \texttt{/usr/local} by default. Check the target
\texttt{install} below for more information.

You can also build other targets from the following list:

\begin{longtable}[]{@{}lll@{}}
\toprule
\begin{minipage}[b]{0.30\columnwidth}\raggedright
Target\strut
\end{minipage} & \begin{minipage}[b]{0.30\columnwidth}\raggedright
Description\strut
\end{minipage} & \begin{minipage}[b]{0.30\columnwidth}\raggedright
Result\strut
\end{minipage}\tabularnewline
\midrule
\endhead
\begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{atc}\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
The ATC library itself.\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{lib/libatc.so}\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{cli}\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
The command-line interface (requires
\href{https://www.boost.org/doc/libs/1_75_0/doc/html/program_options.html}{Boost.Program\_Options}).\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{bin/atc}\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{install}\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
Builds the targets \texttt{atc} and \texttt{cli} (if valid) and copies
the executable, library and public header files (both C and C++) to the
location specified during the CMake configuration process.\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{\$\{CMAKE\_INSTALL\_PREFIX\}/bin/atc},
\texttt{\$\{CMAKE\_INSTALL\_PREFIX\}/lib/libatc.so},
\texttt{\$\{CMAKE\_INSTALL\_PREFIX\}/include/atc.hpp},
\texttt{\$\{CMAKE\_INSTALL\_PREFIX\}/include/atc.h},
\texttt{\$\{CMAKE\_INSTALL\_PREFIX\}/include/atc/*}\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{uninstall}\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
Removes all files generated by \texttt{make\ install} based on the
\texttt{install\_manifest.txt}. Only works on Unix systems and if the
\texttt{install\_manifest.txt} file is still in the build
directory.\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
Removes the results of \texttt{make\ install}\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{test-suite}\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
The C++ test suite.\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{bin/test-suite}\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{example}\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
The C++ example driver.\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{bin/example}\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{example-c}\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
The C example driver.\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{bin/example-c}\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{examples}\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
Both the C++ and C example drivers.\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{bin/example} and \texttt{bin/example-c}\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{doc}\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
The ATC documentation (requires \href{https://www.doxygen.nl}{Python}
{[}7{]}, \href{https://www.doxygen.nl}{Doxygen},
\href{https://www.sphinx-doc.org}{Sphinx} and more, see below).\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{doc/build/html/index.html}, \ldots{}\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{readme}\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
The repository's README (requires
\href{https://www.doxygen.nl}{Python}).\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{README.md}, \ldots{}\strut
\end{minipage}\tabularnewline
\begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{manual}\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
The user manual (requires \href{https://www.doxygen.nl}{Python},
\href{https://pandoc.org/}{Pandoc} and pdflatex).\strut
\end{minipage} & \begin{minipage}[t]{0.30\columnwidth}\raggedright
\texttt{user-manual.pdf}\strut
\end{minipage}\tabularnewline
\bottomrule
\end{longtable}

Note that, unfortunately, compiling the HPTT source file
\texttt{transpose.cpp} can take a long time, leading to full ATC
compilation times of 2-4 minutes. Furthermore, we observed that, when
upgrading g++ from version 7 to version 9 or 10, compilation slowed down
even further to 10-15 minutes. However, since this effect wasn't
observed with other compilers, this seems to be a GCC-specific problem.
Therefore, if you think this takes too long, we recommend using a
different compiler.

\hypertarget{uncompresseddecompressed-data-format}{%
\subsubsection{Uncompressed/decompressed data
format}\label{uncompresseddecompressed-data-format}}

The uncompressed/decompressed data should/will be stored in the
``brick-of-floats'' format. This simply means that the tensor is stored
in a flattened format, with each scalar being stored using its binary
representation in little-endian (e.g.~a single-precision floating-point
number is stored using 4 bytes), without any header or footer (although
in the case of uncompressed data, there can be a header as long as you
skip it properly using the
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/c++/options.html\#_CPPv4N3atc7Options10skip_bytesE}{\texttt{atc::Options.skip\_bytes}}
option). If the
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/c++/options.html\#_CPPv4N3atc7Options13storage_orderE}{\texttt{atc::Options.storage\_order}}
is set to
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/c++/options.html\#_CPPv4N3atc3MSIE}{\texttt{atc::StorageOrder::MSI}}
(default), then the flattened tensor will be interpreted with the last
index varying the fastest (the tensor equivalent of row-major storage).
Conversely, if it is set to
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/c++/options.html\#_CPPv4N3atc3LSIE}{\texttt{atc::StorageOrder::LSI}},
the first index will vary the fastest.

For example, consider a 2x2x2 tensor \texttt{A} with
\texttt{A(i,\ j,\ k)\ =\ 100*i\ +\ 10*j\ +\ k} using 1-based indexing.
When flattened using the MSI storage order, we get the following vector:

\begin{verbatim}
[111, 112, 121, 122, 211, 212, 221, 222]
\end{verbatim}

When flattened using the LSI storage order, we get the following vector:

\begin{verbatim}
[111, 211, 121, 221, 112, 212, 122, 222]
\end{verbatim}

Therefore, in order to achieve good compression, \textbf{it is crucial
to use the correct storage order or else the tensor will be interpreted
in a completely wrong way}. In case you're still unsure after reading
this explanation, you can simply try both storage orders once on a data
set of your choice. On decent-sized smooth data sets, this will make a
drastic difference in terms of compression rate.

\hypertarget{testing-the-installation}{%
\subsection{Testing the installation}\label{testing-the-installation}}

You can test ATC by compiling and executing the example program
(\href{https://gitlab.kuleuven.be/numa/software/atc/-/tree/master/examples/example.cpp}{\texttt{examples/example.cpp}})
using differents compilers and hardware. You can do this with a custom
compiler by running the following commands in the ATC root directory:

\begin{verbatim}
rm -rf build
mkdir build
cd build
cmake -DCMAKE_CXX_COMPILER=<g++|clang|...> ..
make example -j4
\end{verbatim}

The example can then be run using the following commands:

\begin{verbatim}
cd ../bin
./example
\end{verbatim}

The output should be similar to:

\begin{verbatim}
ATC version 1.1.2
Compressed file size: 20709
Compression factor: 3.16461
Target relative error: 0.001
Actual relative error: 0.00101744
\end{verbatim}

where the compression factor is the ratio in between the original file
size and the compressed file size, while the relative error is the
Frobenius norm of the difference in between the original and
decompressed tensor, divided by the Frobenius norm of the original
tensor. You can also test the C example program
(\href{https://gitlab.kuleuven.be/numa/software/atc/-/tree/master/examples/example.c}{\texttt{examples/example.c}})
by replacing the above \texttt{make\ example} command with
\texttt{make\ examples} and then executing \texttt{./example-c} in the
\texttt{bin} directory. If you want to choose the C compiler, make sure
to add the flags
\texttt{-DCMAKE\_C\_COMPILER=\textless{}gcc\textbar{}clang\textbar{}...\textgreater{}}
to the aforementioned \texttt{cmake} command. Note that, because both
targets are configured using the same CMake project, the vendor of both
compilers should be the same in this set-up. Of course you should be
able to compile your own applications using the C interface with a
compiler from a different vendor (the interface is meant to be portable
after all), but it would have to be configured separately.

Finally, you can also build and run the target \texttt{test-suite} to
check if all automated tests execute successfully.

\hypertarget{c-api}{%
\subsubsection{C++ API}\label{c-api}}

You can perform compression and decompression by calling the functions
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/c++/interface.html\#atc-compress}{\texttt{atc::compress}}
and
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/c++/interface.html\#atc-decompress}{\texttt{atc::decompress}}.
These functions are overloaded to cover many different situations. Here
we will demonstrate how to use them to compress and decompress a file:

\begin{verbatim}
atc::compress("path/to/original_data", atc::DataType::float32, {10, 20, 30}, 80,
    "path/to/compressed_data");
\end{verbatim}

This will let ATC compress the original data at
\texttt{path/to/original\_data}, a 10x20x30 tensor of 32-bit
floating-point numbers stored in a raw binary format, with a target PSNR
(i.e.~peak signal-to-noise ratio) of 80 dB and store the result in
\texttt{path/to/compressed\_data}. Since the tensor metadata is stored
in the compressed file, we can simply decompress it to the file
\texttt{path/to/decompressed\_data} with the following function call:

\begin{verbatim}
atc::decompress("path/to/compressed_data", "path/to/decompressed_data");
\end{verbatim}

For a more extensive example, check out the source file
\href{https://gitlab.kuleuven.be/numa/software/atc/-/tree/master/examples/example.cpp}{\texttt{examples/example.cpp}}.
For more information, such as the use of
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/c++/options.html}{\texttt{atc::Options}},
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/c++/results.html}{\texttt{atc::Results}},
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/c++/interface.html\#atc-compress-decompress}{\texttt{atc::compress\_decompress}}
or other API functions, please consult the
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/c++.html}{relevant
documentation}.

\hypertarget{c-api-1}{%
\subsubsection{C API}\label{c-api-1}}

As an alternative to the C++ interface, we also provide a C interface.
To understand when to use or not to use this interface, check the
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/c.html}{relevant
documentation}. The aforementioned compress-and-decompress-files
scenario would be translated as follows:

\begin{verbatim}
// Compression
size_t sizes[3] = {10, 20, 30};
atc_compress_ff("path/to/original_data", ATC_DATA_TYPE_FLOAT32, 3, sizes, 80,
    "path/to/compressed_data"); // "ff" means file-to-file
// Decompression
atc_decompress_ff("path/to/compressed_data", "path/to/decompressed_data");
\end{verbatim}

For a more extensive example, check out the source file
\href{https://gitlab.kuleuven.be/numa/software/atc/-/tree/master/examples/example.c}{\texttt{examples/example.c}}.
For more information, please consult the
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/c.html}{relevant
documentation}.

\newpage

\hypertarget{command-line-interface}{%
\subsubsection{Command-line interface}\label{command-line-interface}}

General compression command:

\begin{verbatim}
./atc --of <original-file> --type <data-type> --sizes <tensor-sizes> --<error-metric>
    <target-error> --cf <compressed-file> [OPTIONS]
\end{verbatim}

General decompression command:

\begin{verbatim}
./atc --cf <compressed-file> --df <decompressed-file> [OPTIONS]
\end{verbatim}

You can also combine compression and decompression into a single
command, allowing you to use the \texttt{-\/-ce} flag to calculate the
compression error:

\begin{verbatim}
./atc --of <original-file> --type <data-type> --sizes <tensor-sizes> --<error-metric>
    <target-error> --cf <compressed-file> --df <decompressed-file> [OPTIONS]
\end{verbatim}

Furthermore, you can use INI config files to store and reuse sets of
command-line options using the \texttt{-\/-config} flag. For examples,
check out the Shell script
\href{https://gitlab.kuleuven.be/numa/software/atc/-/tree/master/examples/example.sh}{\texttt{examples/example.sh}}.
Some common options you may want to set:

\begin{itemize}
\tightlist
\item
  \texttt{-\/-rank-truncation\ \textless{}arg\textgreater{}}: Sets the
  RTMSS parameter (in the range 0-1), which affects the trade-off in
  between speed and rate-distortion. Typically, 0 leads to the highest
  compression rates, while 1 leads to much faster compression (default:
  0.5). For details, check out
  \href{https://doi.org/10.48550/arxiv.2107.01384}{our paper}.
\item
  \texttt{-\/-threads\ \textless{}arg\textgreater{}}: The maximal amount
  of threads that will be used. 0 (default) means that ATC will attempt
  to use the amount of cores available on the system.
\item
  \texttt{-\/-verbose}: Prints a lot of information about the
  compression/decompression process during execution.
\end{itemize}

For more information on the command-line options, run
\texttt{./atc\ -\/-help} or check
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc/cli.html}{the
relevant documentation}.

\hypertarget{python-example}{%
\subsubsection{Python example}\label{python-example}}

Although we do not provide full Python bindings, you can find an example
that performs ATC compression and decompression using
\href{https://docs.python.org/3/library/ctypes.html}{ctypes} at
\href{https://gitlab.kuleuven.be/numa/software/atc/-/tree/master/examples/example.py}{\texttt{examples/example.py}},
which demonstrates how ATC can be dynamically loaded and called through
its C wrapper from other languages. If you want to use ATC in Python,
this example is probably a good starting point.

\hypertarget{documentation}{%
\subsubsection{Documentation}\label{documentation}}

You can find the ATC documentation
\href{https://people.cs.kuleuven.be/~wouter.baert/atc/doc}{here}.
Alternatively, you can generate the ATC documentation yourself if you
have installed the following dependencies:

\begin{itemize}
\tightlist
\item
  \href{https://www.python.org/}{Python}
\item
  \href{https://www.doxygen.nl}{Doxygen}
\item
  \href{https://www.sphinx-doc.org}{Sphinx}
\item
  \href{https://breathe.readthedocs.io}{Breathe}
\item
  \href{https://recommonmark.readthedocs.io}{recommonmark}
\item
  \href{https://pypi.org/project/sphinx-markdown-tables}{sphinx-markdown-tables}
\item
  \href{https://pypi.org/project/sphinx-rtd-theme}{sphinx-rtd-theme}
\end{itemize}

by running the following commands:

\begin{verbatim}
cd build
cmake ..
make doc
\end{verbatim}

The documentation can then be viewed by opening the file
\texttt{doc/build/html/index.html} in a web browser. Note that, due to
software installation limitations, this set-up was only tested using
Sphinx v1.8.5. For newer Sphinx versions, the Markdown compatibility
should probably be handled by
\href{https://myst-parser.readthedocs.io}{MyST} instead of
\href{https://recommonmark.readthedocs.io}{recommonmark}, but this is
not supported at the moment.

To generate the user manual, you can build the target \texttt{manual} as
long as you have installed the following dependencies:

\begin{itemize}
\tightlist
\item
  \href{https://www.doxygen.nl}{Python}
\item
  \href{https://pandoc.org/}{Pandoc}
\item
  pdflatex
\end{itemize}

\hypertarget{benchmarks}{%
\subsection{Benchmarks}\label{benchmarks}}

In our experiments, we found that with default settings, ATC maintains
about the same compression rate as the the Tucker-based
\href{https://github.com/rballester/tthresh}{TTHRESH} compressor, while
providing average speed-ups of 3.5 and 2.2 during compression and
decompression. Furthermore, peak memory usage is about halved. Moreover,
compression can be slightly improved by lowering the
\texttt{rank\_truncation\_max\_sse\_share} parameter at the cost of
speed. Finally, ATC achieves very fine error control, on average
deviating only 1.4\% from the requested compression error. Compared to
the Tucker-based \href{https://gitlab.com/tensors/TuckerMPI}{TuckerMPI}
{[}8{]} compressor, we achieve about 10 times higher compression due to
the lack of quantization and encoding in TuckerMPI.

Compared to non-Tucker-based compressors such as
\href{https://github.com/szcompressor/SZ3}{SZ} {[}9{]},
\href{https://github.com/LLNL/ZFP}{ZFP} {[}10{]} and
\href{https://github.com/LLNL/fpzip}{FPZIP} {[}11{]} (as well as our own
\href{https://www.x265.org/}{x265} wrapper), we also observed that ATC
often outperforms all alternatives in terms of compression rate in the
high-error domain (e.g.~1\%-10\% relative error). However, due to the
expensive Tucker decomposition, ATC takes significantly more time and
memory to run compared to
e.g.~\href{https://github.com/szcompressor/SZ3}{SZ}.

For more detailed experimental results, including an extensive series of
graphs, we refer to \href{https://doi.org/10.48550/arxiv.2107.01384}{our
paper}.

\hypertarget{issues}{%
\subsection{Issues}\label{issues}}

If you find a bug in ATC, please open a
\href{https://gitlab.kuleuven.be/numa/software/atc/-/issues/new}{new
issue} or report it to
\href{mailto:wouter.baert@kuleuven.be}{\nolinkurl{wouter.baert@kuleuven.be}}.

\hypertarget{acknowledgements}{%
\subsection{Acknowledgements}\label{acknowledgements}}

I'd like to thank the authors of
\href{https://github.com/rballester/tthresh}{TTHRESH}, which initially
served as the basis of the ATC implementation. Furthermore, ATC's
quantization and encoding scheme is heavily based on their work.

\hypertarget{citation}{%
\subsection{Citation}\label{citation}}

If you use our work in an academic publication, please cite
\href{https://doi.org/10.48550/arxiv.2107.01384}{our paper} (accepted at
the ACM Transactions on Mathematical Software, pre-print available on
arXiv) using the following BibTeX reference:

\begin{verbatim}
@misc{https://doi.org/10.48550/arxiv.2107.01384,
  doi = {10.48550/ARXIV.2107.01384},
  url = {https://arxiv.org/abs/2107.01384},
  author = {Baert, Wouter and Vannieuwenhoven, Nick},
  keywords = {Mathematical Software (cs.MS), FOS: Computer and information sciences,
    FOS: Computer and information sciences},
  title = {ATC: an Advanced Tucker Compression library for multidimensional data},
  publisher = {arXiv},
  year = {2021},
  copyright = {arXiv.org perpetual, non-exclusive license}
}
\end{verbatim}

\newpage

\hypertarget{references}{%
\subsection{References}\label{references}}

{[}1{]} E. Perlman, R. Burns, Y. Li and C. Meneveau. ``Data Exploration
of Turbulence Simulations using a Database Cluster''. Supercomputing
SC07, ACM, IEEE, 2007.
\href{https://doi.org/10.1145/1362622.1362654}{DOI}

{[}2{]} L. S. Blackford, A. Petitet, R. Pozo, K. Remington, R. C.
Whaley, J. Demmel and others. ``An updated set of basic linear algebra
subprograms (BLAS)''. ACM Transactions on Mathematical Software, 28(2),
135--151, 2002. \href{https://doi.org/10.1145/567806.567807}{DOI}

{[}3{]} L. Dagum and R. Menon. ``OpenMP: an industry standard API for
shared-memory programming''. IEEE Computational Science and Engineering,
vol.~5, no. 1, pp.~46-55, Jan.-March 1998.
\href{https://doi.org/10.1109/99.660313}{DOI}

{[}4{]} M. Frigo and S. G. Johnson. ``The Design and Implementation of
FFTW3''. Proceedings of the IEEE 93 (2), 216--231, 2005.
\href{https://doi.org/10.1109/JPROC.2004.840301}{DOI}

{[}5{]} P. Springer, T. Su and P. Bientinesi. ``HPTT: A High-Performance
Tensor Transposition C++ Library''. Proceedings of the 4th ACM SIGPLAN
International Workshop on Libraries, Languages, and Compilers for Array
Programming, ACM, 2017.
\href{https://doi.org/10.1145/3091966.3091968}{DOI}

{[}6{]} R. Ballester-Ripoll, P. Lindstrom and R. Pajarola. ``TTHRESH:
Tensor Compression for Multidimensional Visual Data''. IEEE Transaction
on Visualization and Computer Graphics, IEEE, 2019.
\href{https://doi.org/10.1109/TVCG.2019.2904063}{DOI}

{[}7{]} G. van Rossum. ``Python tutorial''. Technical Report CS-R9526,
Centrum voor Wiskunde en Informatica (CWI), May 1995.

{[}8{]} G. Ballard and A. Klinvex and T. G. Kolda. ``TuckerMPI: A
Parallel C++/MPI Software Package for Large-scale Data Compression via
the Tucker Tensor Decomposition''. ACM Transactions on Mathematical
Software, 46(2), 2020. \href{https://doi.org/10.1145/3378445}{DOI}

{[}9{]} X. Liang, K. Zhao, S. Di, S. Li, R. Underwood, A. M. Gok, J.
Tian, J. Deng, J. C. Calhoun, D. Tao, Z. Chen, and F. Cappello. ``SZ3: A
modular framework for composing prediction-based error-bounded lossy
compressors''. IEEE Transactions on Big Data.
\href{https://doi.org/10.1109/TBDATA.2022.3201176}{DOI}

{[}10{]} P. Lindstrom. ``Fixed-Rate Compressed Floating-Point Arrays''.
IEEE Transactions on Visualization and Computer Graphics 20, 12,
2674--2683, 2014. \href{https://doi.org/10.1109/TVCG.2014.2346458}{DOI}

{[}11{]} P. Lindstrom and M. Isenburg. ``Fast and Efficient Compression
of Floating-Point Data''. IEEE Transactions on Visualization and
Computer Graphics 12, 5, 1245--1250, 2006.
\href{https://doi.org/10.1109/TVCG.2006.143}{DOI}

\end{document}
