#!/bin/bash
# Copyright (C) 2015-2019  Etersoft
# Copyright (C) 2015-2019  Vitaly Lipatov <lav@etersoft.ru>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

export LANG=C

PROGDIR=$(dirname $0)
[ "$PROGDIR" = "." ] && PROGDIR=$(pwd)

VERSION="0.9"
NEEDZPAQ="7.15"
BASEPACK=root
# metadata files from eterattrstore, kept in the root of the root archive
PAXFILE=.eterpack.special.files.pax
METAFILE=.metadata
FOLLOWDIR=.eterpack.follow
METANAMES=("$METAFILE" "$PAXFILE" "$FOLLOWDIR")
# templates for multi part archive names
STZ="000000"
ST1="000001"
STN="??????"

DESCR="eterpack version $VERSION (c) Etersoft 2015-2019"

# realpath workaround
if ! which realpath 2>/dev/null >/dev/null ; then
realpath()
{
	readlink -f "$@"
}
fi

print_message()
{
	local DESC="$1"
	shift
	echo "$DESC in $(basename -- "$0"): $*"
}

# Print error message and stop the program
fatal()
{
	print_message Error "$@" >&2
	exit 1
}

info()
{
	echo "$*"
}

fatal_status()
{
	if [ -s "$DUMPDIR/.status" ] ; then
		echo "$1" > "$DUMPDIR/.status"
	fi
	if [ -n "$2" ] ; then
		echo "$2" > "$DUMPDIR/.status_ext"
	fi
	fatal "Exited with status $1. $2"
}

done_status()
{
	rm -f "$DUMPDIR/.status"
	rm -f "$DUMPDIR/.status_ext"
	date --iso-8601=minutes -u >"$DUMPDIR/.last-updated"
}

# Note: used egrep! write '[0-9]+(first|two)', not '[0-9]\+...'
rhas()
{
	echo "$1" | grep -E -q -- "$2"
}

# //path//path1 -> path/path1
clean_path()
{
	echo "$1" | sed -e \
		"s|^//||g
		 s|^/||g
		 s|///|/|g
		 s|//|/|g"
}

# return 0 if reldir (with last name bdir) is excluded by any --exclude
# (or it is inside an excluded path)
is_excluded_dir()
{
	local reldir="$1"
	local bdir="$2"
	local ex exdir
	for ex in "${EXCLUDES[@]}" ; do
		if rhas "$ex" "^/" ; then
			exdir="$(clean_path "$ex")"
			[ "$reldir" = "$exdir" ] && return 0
			[ "${reldir#"$exdir"/}" != "$reldir" ] && return 0
		else
			[ "$bdir" = "$ex" ] && return 0
		fi
	done
	return 1
}

list_dirs()
{
local reldir
#echo "List dirs from $BACKUPDIR, with ${EXCLUDES[*]} exclude."
# Получаем список каталогов, которые мы хотим превратить в файлы
find "${FOLLOWLINKS[@]}" "$BACKUPDIR" -depth -maxdepth "$MAXDEPTH" -mindepth "$MINDEPTH" -type d -printf "%P\n" | \
while read -r reldir ; do

	bdir=$(basename -- "$reldir")

	is_excluded_dir "$reldir" "$bdir" && continue

	echo "$reldir"
done
}

list_rootfiles()
{
	# Note: --exclude rules for root files are passed to zpaq as -not
	# with -L, -xtype l is true for symlinks (zpaq skips them, eterattrstore saves them)
	find "${FOLLOWLINKS[@]}" "$BACKUPDIR" -depth -maxdepth "$MAXDEPTH" -mindepth 1 -type f ! -xtype l -printf "%P\n"
	# empty dirs are listed by list_empty_dirs
}

# empty dirs above the packing depth (they are not in any dir archive)
list_empty_dirs()
{
	local reldir
	[ "$MAXDEPTH" -gt 1 ] || return 0
	find "$BACKUPDIR" -mindepth 1 -maxdepth "$((MAXDEPTH-1))" -type d -empty -printf "%P\n" | \
	while read -r reldir ; do
		is_excluded_dir "$reldir" "$(basename -- "$reldir")" && continue
		echo "$reldir"
	done
}

# run zpaq add with -index ZARCH (the first arg), but do not keep a new part without changes:
# zpaq creates a part with an empty version (104 bytes) on every run
zpaq_add()
{
	local zarch="$1" prev='' output res part num
	shift
	# without the index zpaq starts from part 1 again and fails with "archive exists" for it
	if [ ! -e "$zarch" ] && [ -e "${zarch%."$STZ".zpaq}.$ST1.zpaq" ] ; then
		fatal_status "unrecoverable error" "The index $zarch is missing, but the parts exist"
	fi
	if [ -s "$zarch" ] ; then
		prev="$zarch.prev"
		cp -p "$zarch" "$prev" || fatal "Can't copy $zarch"
	fi
	output="$(mktemp)" || fatal
	zpaq add "$@" -index "$zarch" 2>&1 | tee "$output"
	res="${PIPESTATUS[0]}"
	part="$(sed -n -e 's|^Creating \(.*\.zpaq\) at offset .*|\1|p' "$output")"
	# zpaq numbers a new part by the versions in the index, so missed parts leave a gap
	# and zpaq does not read the parts after it
	# (without part 1 all the parts are moved away, zpaq supports it with the index)
	num="$(echo "$part" | sed -n -e 's|.*\.0*\([0-9][0-9]*\)\.zpaq$|\1|p')"
	if [ "$res" = "0" ] && [ -n "$num" ] && [ "$num" -gt 1 ] && \
	   [ -e "${zarch%."$STZ".zpaq}.$ST1.zpaq" ] && \
	   [ ! -e "$(printf '%s.%06d.zpaq' "${zarch%."$STZ".zpaq}" "$((num - 1))")" ] ; then
		[ -n "$prev" ] && mv -f "$prev" "$zarch"
		rm -f "$part" "$output"
		fatal_status "unrecoverable error" "Parts before $part are missing (the index has more versions than the parts)"
	fi
	if [ "$res" = "0" ] && [ -n "$prev" ] && grep -q -x "0 +added, 0 -removed\." "$output" ; then
		if [ -f "$part" ] ; then
			# restore the index first: if we fail before removing the part,
			# the next run gets "archive exists" for it and moves it to .broken
			mv -f "$prev" "$zarch" && rm -f "$part" && echo "No changes, $part is removed"
		fi
	fi
	rm -f "$prev" "$output"
	return "$res"
}

attrstore()
{
	$PROGDIR/eterattrstore "$@"
}

# symlinks to dirs followed with --followlinks (packed as dirs under the link names)
list_followed_links()
{
	local reldir
	[ -n "${FOLLOWLINKS[*]}" ] || return 0
	find -L "$BACKUPDIR" -mindepth 1 -maxdepth "$MAXDEPTH" -type d -printf "%P\n" | \
	while read -r reldir ; do
		[ -L "$BACKUPDIR/$reldir" ] && echo "$reldir"
	done
	return 0
}

# extract the metafiles stored in the root archive (internal names have prefix $2) to dir $1
extract_stored_metafiles()
{
	local outdir="$1" prefix="$2" name
	local INNAMES=() OUTNAMES=()
	for name in "${METANAMES[@]}" ; do
		INNAMES+=("$prefix$name")
		OUTNAMES+=("$outdir/$name")
	done
	# zpaq restores the stored dates
	zpaq extract "$DUMPDIR/$BASEPACK.$STN.zpaq" "${INNAMES[@]}" -to "${OUTNAMES[@]}" "${PASSWORDARG[@]}" >/dev/null 2>&1
	return 0
}

# set the stored date to the metafiles in dir $1 which are the same as in the root archive,
# so zpaq does not add a new version for them (internal names have prefix $2)
keep_stored_metadates()
{
	local metadir="$1" prefix="$2" stored f
	[ -s "$DUMPDIR/$BASEPACK.$STZ.zpaq" ] || return 0
	stored="$(mktemp -d "$DUMPDIR/.eterpack.tmp.XXXXXX")" || return 0
	extract_stored_metafiles "$stored" "$prefix"
	while read -r f ; do
		[ -f "$stored/$f" ] && cmp -s "$metadir/$f" "$stored/$f" && touch -r "$stored/$f" "$metadir/$f"
	done < <(cd "$metadir" && find . -type f -printf "%P\n")
	rm -rf "$stored"
	return 0
}

# paths (relative to the source root) excluded by --exclude
list_excluded_paths()
{
	local ex reldir
	for ex in "${EXCLUDES[@]}" ; do
		# with --depth 0 names are paths from the root too
		if rhas "$ex" "^/" || [ "$MAXDEPTH" = "0" ] ; then
			clean_path "$ex"
		fi
	done
	[ "$MAXDEPTH" = "0" ] && return
	# names are matched at the packing depth
	find "${FOLLOWLINKS[@]}" "$BACKUPDIR" -mindepth "$MINDEPTH" -maxdepth "$MAXDEPTH" -type d -printf "%P\n" | \
	while read -r reldir ; do
		is_excluded_dir "$reldir" "$(basename -- "$reldir")" && echo "$reldir"
	done
	return 0
}

# save attrs and special files of the source to a new dir in DUMPDIR, print the dir
# (the source tree is not changed), $1 is the prefix of internal names
save_metadir()
{
	local metadir
	metadir="$(mktemp -d "$DUMPDIR/.eterpack.tmp.XXXXXX")" || fatal "Can't create a temp dir in $DUMPDIR"
	list_followed_links >"$metadir.follow"
	list_excluded_paths >"$metadir.exclude"
	(cd "$BACKUPDIR" && attrstore save --metadir "$metadir" --follow "$metadir.follow" --exclude "$metadir.exclude") || \
		{ rm -rf "$metadir" "$metadir.follow" "$metadir.exclude" ; fatal "Can't save attributes and special files" ; }
	rm -f "$metadir.follow" "$metadir.exclude"
	keep_stored_metadates "$metadir" "$1"
	echo "$metadir"
}

# fatal if the source has a file with our reserved names
assert_no_metanames()
{
	local name
	for name in "${METANAMES[@]}" ; do
		if [ -e "$BACKUPDIR/$name" ] || [ -L "$BACKUPDIR/$name" ] ; then
			fatal "$BACKUPDIR/$name is reserved for eterpack metadata, rename it or use --noattribute"
		fi
	done
}

# direct run for single zpaq archive
update_archive()
{
	bdir="$(basename -- "$BACKUPDIR")"

	local metadir='' FILES=("./$bdir") TONAMES=(.) name
	if [ -z "$NOATTRIBUTE" ] ; then
		echo
		echo "Save attrs and special files..."
		metadir="$(save_metadir ./)" || exit
		# missed metafiles are marked as removed by zpaq itself (they are under ./$bdir)
		for name in "${METANAMES[@]}" ; do
			[ -e "$metadir/$name" ] || continue
			FILES+=("$metadir/$name")
			TONAMES+=("./$name")
		done
	fi

	local EXCLUDEARG=()
	local ex
	for ex in "${EXCLUDES[@]}" ; do
		EXCLUDEARG+=(-not "./$bdir/$(clean_path "$ex")")
	done

	cd "$BACKUPDIR/.." || fatal
	echo
	echo "Packing single $BACKUPDIR ..."
	mkdir -p "$DUMPDIR/"
	echo zpaq add "$DUMPDIR/$BASEPACK.$STN.zpaq" "${FILES[@]}" -to "${TONAMES[@]}" "${CHECKSUM[@]}" "${EXCLUDEARG[@]}" -index "$DUMPDIR/$BASEPACK.$STZ.zpaq"
	zpaq_add "$DUMPDIR/$BASEPACK.$STZ.zpaq" "$DUMPDIR/$BASEPACK.$STN.zpaq" "${FILES[@]}" -to "${TONAMES[@]}" "${CHECKSUM[@]}" "${EXCLUDEARG[@]}" "${PASSWORDARG[@]}" || \
		{ [ -n "$metadir" ] && rm -rf "$metadir" ; exit 1 ; }
	cd - >/dev/null || exit

	[ -n "$metadir" ] && rm -rf "$metadir"

	if [ -n "$EXECUTEAFTER" ] ; then
		$EXECUTEAFTER "$DUMPDIR/$BASEPACK.$STN.zpaq" || fatal "Executed command $EXECUTEAFTER failed"
	fi

}

# print the names stored in the last version of archive $1, without the trailing /
list_archive_names()
{
	zpaq list "$1" "${PASSWORDARG[@]}" | \
		sed -n -e 's|^- [0-9-]* [0-9:]* *[0-9]* *[^ ]* \(.*\)$|\1|p' | sed -e 's|/$||'
}

# names from the root archive which are removed from the source
list_removed_rootnames()
{
	local name
	[ -s "$DUMPDIR/$BASEPACK.$STZ.zpaq" ] || return 0
	list_archive_names "$DUMPDIR/$BASEPACK.$STN.zpaq" | while read -r name ; do
		# metafiles are not in the source
		[ -z "$NOATTRIBUTE" ] && printf '%s\n' "${METANAMES[@]}" | grep -q -x -F -e "$name" -e "${name%%/*}" && continue
		[ -e "$BACKUPDIR/$name" ] || [ -L "$BACKUPDIR/$name" ] || echo "$name"
	done
}

# mark as removed the dir archives for the dirs which are removed from the source
# (zpaq add marks the stored files as removed if they are missed)
mark_removed_dirs()
{
	local relfile reldir bdir intname emptydir
	emptydir="$(mktemp -d)" || fatal
	while read -r relfile ; do
		reldir="$(dirname -- "$relfile")"
		[ "$reldir" = "." ] && continue
		[ -d "$BACKUPDIR/$reldir" ] && continue
		# keep the backup of excluded dirs
		is_excluded_dir "$reldir" "$(basename -- "$reldir")" && continue
		bdir="$(basename -- "$reldir")"
		intname="$bdir"
		rhas "$intname" "^-" && intname="./$intname"
		local archname="$DUMPDIR/$reldir/$bdir.$STN.zpaq"
		local zarchname="$DUMPDIR/$reldir/$bdir.$STZ.zpaq"
		# no files with the archived names after the last update
		list_archive_names "$archname" | grep -q . || continue

		echo
		echo "Mark removed $reldir in $archname ..."
		(cd "$emptydir" && zpaq_add "$zarchname" "$archname" "./$bdir" -to "$intname" "${PASSWORDARG[@]}") || \
			fatal_status "unrecoverable error" "Can't mark removed $reldir in $archname"

		if [ -n "$EXECUTEAFTER" ] ; then
			$EXECUTEAFTER "$archname" || fatal "Executed command $EXECUTEAFTER failed"
		fi
	done < <(find "$DUMPDIR" -type f -name "*.$STZ.zpaq" -printf "%P\n" | sed -e "s|\.$STZ.zpaq$|.$STN.zpaq|g")
	rmdir "$emptydir"
}

update_dump()
{
local reldir
# Получаем список каталогов, которые мы хотим превратить в файлы
find "${FOLLOWLINKS[@]}" "$BACKUPDIR" -depth -maxdepth "$MAXDEPTH" -mindepth "$MINDEPTH" -type d -printf "%P\n" | \
while read -r reldir ; do

	bdir="$(basename -- "$reldir")"

	local EXCLUDEREL=()
	local ex skipdir=''
	for ex in "${EXCLUDES[@]}" ; do
		if rhas "$ex" "^/" ; then
			if [ "$reldir" = "$(clean_path "$ex")" ] ; then
				echo "Skip excluded $reldir"
				skipdir=1
				break
			fi
			if rhas "$reldir" "^$(clean_path "$ex")/" ; then
				echo "Skip excluded $reldir (part)"
				skipdir=1
				break
			fi
			if rhas "$ex" "^/$reldir/" ; then
				EXCLUDEREL+=("$(clean_path "$(echo "$ex" | sed -e "s|^/$reldir||g")")")
			fi
		else
			if [ "$bdir" = "$ex" ] ; then
				echo "Skip excluded $bdir"
				skipdir=1
				break
			fi
		fi
	done
	[ -n "$skipdir" ] && continue

	# zpaq does not support symlink, so workaround it
	if [ -n "${FOLLOWLINKS[*]}" ] ; then
		babsdir="$(basename -- "$(realpath "$BACKUPDIR/$reldir")")"
		babsreldir="$(realpath "$BACKUPDIR/$reldir/..")"
	else
		babsdir="$bdir"
		babsreldir="$BACKUPDIR/$reldir/.."
	fi

	# workaround for correct internal path
	cd "$babsreldir" || exit

	# exclude rules match the external path
	local EXCLUDEARG=()
	for ex in "${EXCLUDEREL[@]}" ; do
		EXCLUDEARG+=(-not "./$babsdir/$ex")
	done
	[ -n "${EXCLUDEARG[*]}" ] && echo "Use exclude rule: ${EXCLUDEARG[*]}"

	# internal path is the dir path in the source tree (the link name for --followlinks)
	local intname="$bdir"
	if [ -n "$SINGLEARCHIVE" ] ; then
		intname="$reldir"
		local archname="$DUMPDIR/$BASEPACK.$STN.zpaq"
		local zarchname="$DUMPDIR/$BASEPACK.$STZ.zpaq"
	else
		mkdir -p "$DUMPDIR/$reldir"

		local archname="$DUMPDIR/$reldir/$bdir.$STN.zpaq"
		local zarchname="$DUMPDIR/$reldir/$bdir.$STZ.zpaq"
	fi
	# zpaq takes a name started with '-' as an option
	rhas "$intname" "^-" && intname="./$intname"

	echo
	echo "Packing $reldir to $archname ..."

	local tmpoutput="$(mktemp)"
	# Note: ./ protects names started with '-'
	echo zpaq add "$archname" "./$babsdir" -to "$intname" "${CHECKSUM[@]}" "${EXCLUDEARG[@]}" -index "$zarchname"
	(zpaq_add "$zarchname" "$archname" "./$babsdir" -to "$intname" "${CHECKSUM[@]}" "${EXCLUDEARG[@]}" "${PASSWORDARG[@]}" 2>&1 && touch "$tmpoutput.ok") | tee "$tmpoutput"
	local res=0 existfile='' skipfile=''
	if [ ! -r "$tmpoutput.ok" ] ; then
		res=1
		existfile="$(grep "zpaq: archive exists" "$tmpoutput" | sed -e 's|^\(/.*\.zpaq\).*|\1|g')"
		skipfile="$(grep "zpaq: Skipping block" "$tmpoutput" | sed -e 's|^\(/.*\.zpaq\).*|\1|g')"
	fi
	rm -f "$tmpoutput" "$tmpoutput.ok"

	if [ -n "$existfile" ] ; then
		mv -v "$existfile" "$existfile.broken" || fatal "can't move"
		fatal_status "recoverable error" "Run again to continue"
	fi
	if [ -n "$skipfile" ] ; then
		info "Ignore error with $skipfile ..."
		res=0
	fi
	[ "$res" = "0" ] || fatal_status "unrecoverable error" "Error during packaging to $archname"

	cd - >/dev/null || exit

	if [ -n "$EXECUTEAFTER" ] ; then
		$EXECUTEAFTER "$archname" || fatal "Executed command $EXECUTEAFTER failed"
	fi
done || return

mark_removed_dirs

local FILES=()
# Note: ./ protects names started with '-'
# removed names are passed too, so zpaq marks them as removed
mapfile -t FILES < <( (list_rootfiles ; list_empty_dirs ; list_removed_rootnames) | sed -e 's|^-|./-|')

# internal names are the same as the external ones, except the metadata dir
local TONAMES=("${FILES[@]}")

local metadir='' name
if [ -z "$NOATTRIBUTE" ] ; then
	echo
	echo "Save attrs and special files..."
	metadir="$(save_metadir)" || exit
	local stored=''
	[ -s "$DUMPDIR/$BASEPACK.$STZ.zpaq" ] && stored="$(list_archive_names "$DUMPDIR/$BASEPACK.$STN.zpaq")"
	for name in "${METANAMES[@]}" ; do
		# pass the missed metafiles stored before, so zpaq marks them as removed
		[ -e "$metadir/$name" ] || printf '%s\n' "$stored" | grep -q -x -F -e "$name" || continue
		FILES+=("$metadir/$name")
		TONAMES+=("$name")
	done
fi

# workaround for correct internal path
cd "$BACKUPDIR" || exit

if [ -n "${FILES[*]}" ] ; then

	mkdir -p "$DUMPDIR"

	local EXCLUDEARG=()
	local ex
	for ex in "${EXCLUDES[@]}" ; do
		rhas "$ex" "^/" && EXCLUDEARG+=(-not "$(clean_path "$ex")")
	done

	echo
	echo "Packing extra files ..."
	echo zpaq add "$DUMPDIR/$BASEPACK.$STN.zpaq" "${FILES[@]}" -to "${TONAMES[@]}" "${CHECKSUM[@]}" "${EXCLUDEARG[@]}" -index "$DUMPDIR/$BASEPACK.$STZ.zpaq"
	zpaq_add "$DUMPDIR/$BASEPACK.$STZ.zpaq" "$DUMPDIR/$BASEPACK.$STN.zpaq" "${FILES[@]}" -to "${TONAMES[@]}" "${CHECKSUM[@]}" "${EXCLUDEARG[@]}" "${PASSWORDARG[@]}" || \
		{ [ -n "$metadir" ] && rm -rf "$metadir" ; cd - >/dev/null || exit ; return 1 ; }

	[ -n "$metadir" ] && rm -rf "$metadir"

	if [ -n "$EXECUTEAFTER" ] ; then
		$EXECUTEAFTER "$DUMPDIR/$BASEPACK.$STN.zpaq" || fatal "Executed command $EXECUTEAFTER failed"
	fi
fi

cd - >/dev/null || exit

}

# fatal if there are no archives in $DUMPDIR
assert_archives()
{
	[ -n "$(find "$DUMPDIR" -type f -name "*.$STZ.zpaq" -print -quit)" ] || fatal "No zpaq archives are found in $DUMPDIR"
}

extract_dump()
{
local relfile
assert_archives
mkdir -p "$DESTDIR" || fatal "Can't create output directory $DESTDIR"
# Note: it is important to use ???.zpaq during extract (000.zpaq has no files, 001.zpaq will unpack only this archive).
find "$DUMPDIR" -type f -name "*.$STZ.zpaq" -printf "%P\n" | sed -e "s|\.$STZ.zpaq$|.$STN.zpaq|g" | \
while read -r relfile ; do
	reldir="$(dirname -- "$relfile")"
	# TODO: хранить структуру каталогов отдельно, сразу и проверка?

	# workaround for root archive
	if [ "$reldir" = "." ] ; then
		tdir="$DESTDIR"
	else
		tdir="$(realpath -m "$DESTDIR/$reldir/..")"
	fi

	echo
	echo "Extract files from $relfile to $tdir ..."
	zpaq extract "$DUMPDIR/$relfile" -to "$tdir" "${ROLLBACK[@]}" "${PASSWORDARG[@]}" || exit
done || fatal "Can't extract from $DUMPDIR"

	if [ -z "$NOATTRIBUTE" ] ; then
		cd "$DESTDIR" || return
		echo
		echo "Restore attrs ..."
		# move the metadata out of the tree first: the restored root dir can be read-only
		local metadir name
		metadir="$(mktemp -d)" || fatal
		for name in "${METANAMES[@]}" ; do
			if [ -e "$name" ] ; then
				mv "$name" "$metadir/" || fatal "Can't move $DESTDIR/$name"
			fi
		done
		attrstore restore --metadir "$metadir"
		attrstore clear --metadir "$metadir"
		cd - >/dev/null || exit
	fi
}

# compare archive $1 with files in dir $2 (the dir where the archive is extracted to)
# the rest args are extra zpaq options
# return 1 if there are differences
compare_archive()
{
	local archive="$1" dir="$2"
	shift 2
	local names=() skipargs=() grepargs=(-e "" -e ".") n
	# skip our metafiles
	for n in "${METANAMES[@]}" ; do
		grepargs+=(-e "$n" -e "./$n")
		skipargs+=(-not "$n" -not "./$n")
	done
	# top level names stored in the archive, without our metafiles
	# Note: names are stored with ./ for --depth 0 and for names started with '-'
	mapfile -t names < <(list_archive_names "$archive" | \
		sed -E -e 's|^(\./[^/]+).*|\1|' -e 't' -e 's|/.*||' | \
		grep -v -x -F "${grepargs[@]}" | sort -u)
	[ -n "${names[*]}" ] || return 0

	local output
	output="$(cd "$dir" && zpaq list "$archive" "${names[@]}" "${PASSWORDARG[@]}" -not = "${CHECKSUM[@]}" \
		"${skipargs[@]}" "$@")" || fatal "Can't compare $archive"
	echo "$output"

	# '#' internal file differs (followed by '+' with the external one), '+' external only, '-' internal only
	# skip dirs: dir dates are not restored and archives can keep files without their dirs
	echo "$output" | awk '
		{ name = $0 ; sub(/^[-#+] [0-9-]+ [0-9:]+ +[0-9]+ +[^ ]* /, "", name) }
		name ~ /\/$/ { next }
		/^# [0-9]/ { changed[name] = 1 ; diff++ ; next }
		/^\+ [0-9]/ { if (!(name in changed)) diff++ ; next }
		/^- [0-9]/ { diff++ }
		END { exit diff > 0 }'
}

# compare owner, group, mode and special files stored by eterattrstore with $DESTDIR
compare_metadata()
{
	local metadir res=0
	[ -s "$DUMPDIR/$BASEPACK.$STZ.zpaq" ] || return 0
	metadir="$(mktemp -d)" || fatal
	extract_stored_metafiles "$metadir" ""
	# --depth 0 stores them with ./
	[ -n "$(ls -A "$metadir")" ] || extract_stored_metafiles "$metadir" "./"
	if [ -n "$(ls -A "$metadir")" ] ; then
		echo
		echo "Compare owner, group, mode and special files with $DESTDIR ..."
		# skip special files in the excluded paths (exclude names can be at any depth here)
		local ex
		for ex in "${EXCLUDES[@]}" ; do
			if rhas "$ex" "^/" ; then
				clean_path "$ex"
			else
				find "$DESTDIR" -type d -name "$ex" -printf "%P\n"
			fi
		done >"$metadir.exclude"
		(cd "$DESTDIR" && attrstore compare --metadir "$metadir" --exclude "$metadir.exclude") || res=1
		rm -f "$metadir.exclude"
	fi
	rm -rf "$metadir"
	return $res
}

compare_dump()
{
	local relfile res=0
	assert_archives
	[ -d "$DESTDIR" ] || fatal "Can't open directory $DESTDIR"

	# dirs packed to separate archives are skipped when comparing the root archive
	local skipdirs=() d
	while read -r d ; do
		d="$(dirname -- "$d")"
		# zpaq takes '-' as an option, such dirs can be at the top level only and need no skipping
		[ "$d" = "." ] || rhas "$d" "^-" || skipdirs+=(-not "$d")
	done < <(find "$DUMPDIR" -type f -name "*.$STZ.zpaq" -printf "%P\n")

	while read -r relfile ; do
		reldir="$(dirname -- "$relfile")"

		echo
		# workaround for root archive
		if [ "$reldir" = "." ] ; then
			echo "Compare files from $relfile with $DESTDIR ..."
			compare_archive "$DUMPDIR/$relfile" "$DESTDIR" "${skipdirs[@]}" || res=1
		else
			tdir="$(realpath -m "$DESTDIR/$reldir/..")"
			echo "Compare files from $relfile with $tdir ..."
			compare_archive "$DUMPDIR/$relfile" "$tdir" || res=1
		fi
	done < <(find "$DUMPDIR" -type f -name "*.$STZ.zpaq" -printf "%P\n" | sed -e "s|\.$STZ.zpaq$|.$STN.zpaq|g")

	[ -n "$NOATTRIBUTE" ] || compare_metadata || res=1

	echo
	if [ "$res" = 0 ] ; then
		echo "No differences are found."
	else
		echo "There are differences."
	fi
	return $res
}

test_dump()
{
	local relfile
	assert_archives
	# Note: it is important to use ???.zpaq during extract (000.zpaq has no files, 001.zpaq will unpack only this archive).
	find "$DUMPDIR" -type f -name "*.$STZ.zpaq" -printf "%P\n" | sed -e "s|\.$STZ.zpaq$|.$STN.zpaq|g" | \
	while read -r relfile ; do
		reldir="$(dirname -- "$relfile")"

		echo
		echo "Checking $relfile ..."
		zpaq extract "$DUMPDIR/$relfile" -test "${PASSWORDARG[@]}" || exit
	done || return

	check_dirs_list
}

# check that every dir from dirs.list (the dirs packed by the last update) has its archive
# (or is stored in the root archive with --singlearchive)
check_dirs_list()
{
	local reldir rootnames='' res=0
	[ -s "$DUMPDIR/dirs.list" ] || return 0
	echo
	echo "Checking the archives for dirs from dirs.list ..."
	[ -s "$DUMPDIR/$BASEPACK.$STZ.zpaq" ] && rootnames="$(list_archive_names "$DUMPDIR/$BASEPACK.$STN.zpaq")"
	while read -r reldir ; do
		[ -s "$DUMPDIR/$reldir/$(basename -- "$reldir").$STZ.zpaq" ] && continue
		printf '%s\n' "$rootnames" | grep -q -x -F -e "$reldir" -e "./$reldir" && continue
		echo "Missing archive for $reldir"
		res=1
	done <"$DUMPDIR/dirs.list"
	return $res
}

# move the compacted parts from $1 (with .ready file) to the archive with base path $2
swap_compacted()
{
	local tmp="$1" base="$2" count n p
	read -r count <"$tmp/.ready" || fatal "Can't read $tmp/.ready"
	for ((n = 1; n <= count; n++)) ; do
		p="$(printf '%s.%06d.zpaq' "$(basename -- "$base")" "$n")"
		[ -e "$tmp/$p" ] || continue
		# the part is not renumbered
		if [ "$tmp/$p" -ef "$(dirname -- "$base")/$p" ] ; then
			rm -f "$tmp/$p"
			continue
		fi
		mv -f "$tmp/$p" "$(dirname -- "$base")/$p" || fatal "Can't move $tmp/$p"
	done
	# remove the rest old parts
	for ((n = count + 1; ; n++)) ; do
		p="$(printf '%s.%06d.zpaq' "$base" "$n")"
		[ -e "$p" ] || break
		rm -f "$p" || fatal "Can't remove $p"
	done
	mv -f "$tmp/index" "$base.$STZ.zpaq" || fatal "Can't move $tmp/index"
	rm -rf "$tmp"
}

# remove empty parts (with versions without changes) of archive with index $1 and renumber the rest
compact_archive()
{
	local zarch="$1" base name tmp n p size
	base="${zarch%."$STZ".zpaq}"
	name="$(basename -- "$base")"
	tmp="$(dirname -- "$base")/.eterpack.compact.$name"

	if [ -d "$tmp" ] ; then
		if [ -s "$tmp/.ready" ] ; then
			echo "Finish interrupted compact of $base ..."
			swap_compacted "$tmp" "$base"
			return
		fi
		rm -rf "$tmp"
	fi

	local parts=()
	for ((n = 1; ; n++)) ; do
		p="$(printf '%s.%06d.zpaq' "$base" "$n")"
		[ -e "$p" ] || break
		parts+=("$p")
	done
	# shellcheck disable=SC2206 # glob is wanted here
	local all=("$base".$STN.zpaq)
	if [ "${#all[@]}" != "$((${#parts[@]} + 1))" ] ; then
		info "Skip $base: there is a gap in the part numbers"
		return 1
	fi

	# versions without changes
	local list empty=()
	list="$(zpaq list "$base.$STN.zpaq" -all 6 2>&1)" || { info "Skip $base: can't list it" ; return 1 ; }
	mapfile -t empty < <(echo "$list" | sed -n -E -e 's|^- .* 0*([0-9]+)/ \+0 -0 -> 104$|\1|p')

	# 0 bytes parts have no versions
	local keep=() v=0 dropped=0
	for p in "${parts[@]}" ; do
		size="$(stat -c %s -- "$p")"
		if [ "$size" != "0" ] ; then
			v=$((v + 1))
			if [ "$size" != "104" ] || ! printf '%s\n' "${empty[@]}" | grep -q -x -F -e "$v" ; then
				keep+=("$p")
				continue
			fi
		fi
		dropped=$((dropped + 1))
	done
	[ "$dropped" = "0" ] && return 0

	mkdir "$tmp" || fatal "Can't create $tmp"
	n=0
	for p in "${keep[@]}" ; do
		n=$((n + 1))
		ln -- "$p" "$tmp/$(printf '%s.%06d.zpaq' "$name" "$n")" || fatal "Can't link $p"
	done
	(cd "$tmp" && zpaq extract "$tmp/$name.$STN.zpaq" -index "$tmp/index" >/dev/null) || fatal "Can't create the index for $base"
	# check the result has all versions with changes
	local versions
	versions="$(zpaq list "$tmp/$name.$STN.zpaq" -all 6 2>/dev/null | grep -c -E '^- .* [0-9]+/ ')"
	[ "$versions" = "$((v - ${#empty[@]}))" ] || fatal "Compacted $base has $versions versions instead of $((v - ${#empty[@]}))"

	echo "$n" >"$tmp/.ready" || fatal
	swap_compacted "$tmp" "$base"
	echo "Compacted $base: removed $dropped empty parts"
}

compact_dump()
{
	local zarch res=0
	assert_archives
	[ -e "$DUMPDIR/.encrypted" ] && fatal "Encrypted archives can't be compacted (zpaq encryption depends on the part offsets)"
	while read -r zarch ; do
		compact_archive "$DUMPDIR/$zarch" || res=1
	done < <(find "$DUMPDIR" -type f -name "*.$STZ.zpaq" -printf "%P\n")
	return $res
}

# print versions of the archive with index $1: number date time +added -removed size
list_versions()
{
	zpaq list "$1" -all "${PASSWORDARG[@]}" 2>/dev/null | \
		sed -n -E -e 's|^- ([0-9-]+ [0-9:]+) +[0-9]+ +0*([0-9]+)/ (\+[0-9]+ -[0-9]+) -> ([0-9]+)$|\2 \1 \3 \4|p'
}

# print archives of the backup with their versions and sizes
print_info()
{
	local zarch base versions first last parts size
	if [ -e "$DUMPDIR/.status" ] ; then
		echo "Backup $DUMPDIR is incomplete, current status: $(cat "$DUMPDIR/.status")"
	elif [ -s "$DUMPDIR/.last-updated" ] ; then
		echo "Backup $DUMPDIR, last updated at $(cat "$DUMPDIR/.last-updated")"
	else
		echo "Backup $DUMPDIR"
	fi
	[ -e "$DUMPDIR/.encrypted" ] && echo "Encrypted"
	assert_archives
	echo
	while read -r zarch ; do
		base="${zarch%."$STZ".zpaq}"
		# shellcheck disable=SC2206 # glob is wanted here
		local all=("$DUMPDIR/$base".$STN.zpaq)
		parts=$((${#all[@]} - 1))
		size="$(du -c -b -- "${all[@]}" | tail -n1 | cut -f1)"
		# the index has all versions (the parts can be moved away)
		versions="$(list_versions "$DUMPDIR/$zarch")"
		if [ -z "$versions" ] ; then
			echo "$base: $parts parts, $size bytes, can't list versions"
			[ -e "$DUMPDIR/.encrypted" ] && [ -z "${PASSWORDARG[*]}" ] && echo "  (use --password)"
			continue
		fi
		first="$(echo "$versions" | head -n1 | cut -d' ' -f2,3)"
		last="$(echo "$versions" | tail -n1 | cut -d' ' -f2,3)"
		echo "$base: $(echo "$versions" | wc -l) versions ($first - $last), $parts parts, $size bytes"
		if [ -n "$INFOALL" ] ; then
			echo "$versions" | sed -e 's|^|  |'
		fi
	done < <(find "$DUMPDIR" -type f -name "*.$STZ.zpaq" -printf "%P\n" | sort)
}

print_status()
{
	if [ -e "$DUMPDIR/.status" ] ; then
		echo "Archive is incomplete, current status: $(cat "$DUMPDIR/.status")"
		exit 1
	fi
	echo "Archive is completed (last updated at $(cat "$DUMPDIR/.last-updated"))"
	exit 0
}

# zpaq version
ZPAQVERSION=$(zpaq | head -n1 | sed -e "s|.* v\([0-9]\.[0-9]*\) .*|\1|")
[ "$ZPAQVERSION" = "$NEEDZPAQ" ] || fatal "eterpack $VERSION supports only zpaq v$NEEDZPAQ"


COMMAND="$1"
[ -n "$COMMAND" ] && shift

MINDEPTH=1
MAXDEPTH=1
EXCLUDES=()
EXECUTEAFTER=''
CHECKSUM=()
PASSWORDARG=()
NOATTRIBUTE=''
SINGLEARCHIVE=''
FOLLOWLINKS=()
ROLLBACK=()
INFOALL=''

while true ; do
	case "$1" in
	--depth)
		shift
		MINDEPTH=$1
		MAXDEPTH=$1
		shift
		;;

	--exclude)
		shift
		[ -n "$1" ] || fatal "--exclude requires a name"
		EXCLUDES+=("$1")
		shift
		;;

	--exclude-from)
		shift
		[ -r "$1" ] || fatal "Can't read exclude file '$1'"
		while IFS= read -r ex || [ -n "$ex" ] ; do
			# skip empty lines and comments
			[ -z "$ex" ] || rhas "$ex" "^#" || EXCLUDES+=("$ex")
		done <"$1"
		shift
		;;

	--all)
		shift
		INFOALL=1
		;;

	--execute)
		shift
		EXECUTEAFTER="$1"
		shift
		;;

	--checksum)
		shift
		CHECKSUM=(-force)
		;;

	--password)
		shift
		PASSWORDARG=(-key "$1")
		shift
		;;

	--noattribute|--noattributes)
		shift
		NOATTRIBUTE=1
		;;

	--singlearchive)
		shift
		SINGLEARCHIVE="1"
		;;

	--followlinks)
		shift
		FOLLOWLINKS=(-L)
		;;

	--rollback)
		shift
		ROLLBACK=(-until "$1")
		shift
		;;

	*)
		break
		;;
esac
done

case $COMMAND in
	update|backup)
		# from
		BACKUPDIR="$(realpath -e "$1")" || exit

		# to
		DUMPDIR="$(realpath -m "$2")"

		# Note: without -p, do not create full path in suddenly place
		[ -d "$DUMPDIR" ] || mkdir "$DUMPDIR" || exit

		# do not run two updates of the same backup at once
		exec 9>"$DUMPDIR/.lock" || fatal "Can't create $DUMPDIR/.lock"
		flock -n 9 || fatal "Another eterpack is updating $DUMPDIR"
		[ -z "$(find "$DUMPDIR" -name ".eterpack.compact.*" -print -quit)" ] || \
			fatal "There is an interrupted compact in $DUMPDIR, run eterpack compact again"

		info "eterpack: packing to $DUMPDIR ..."

		if [ -s "$DUMPDIR/.status" ] ; then
			info "Previous interrupted task had status $(cat "$DUMPDIR/.status"), continue it."
		fi

		[ -n "$NOATTRIBUTE" ] || assert_no_metanames

		echo "in progress" >"$DUMPDIR/.status"
		[ -n "${PASSWORDARG[*]}" ] && echo "encrypted" >"$DUMPDIR/.encrypted"

		if [ "$MAXDEPTH" = "0" ] ; then
			update_archive
			done_status
			exit
		fi

		update_dump || exit
		# TODO: это второй прогон. Надо встроить внутрь. Но там будет дописывание в файл
		list_dirs >"$DUMPDIR/dirs.list"
		list_rootfiles >"$DUMPDIR/files.list"
		done_status
	;;
	repack)
		fatal "do not realized"
		# from
		BACKUPDIR="$(realpath -e "$1")" || exit

		# to
		DUMPDIR="$(realpath -m "$2")"
		repack_dump
	;;
	compare)
		DUMPDIR="$(realpath -e "$1")"

		DESTDIR="$(realpath -e "$2")"
		compare_dump
	;;
	inlist)
		# from
		BACKUPDIR="$(realpath -e "$1")" || exit
		echo "Dirs:"
		list_dirs
		echo
		echo "Files:"
		list_rootfiles
	;;
	extract|restore)
		DUMPDIR="$(realpath -e "$1")" || exit
		[ -d "$DUMPDIR" ] || fatal "$DUMPDIR is not a dir"
		# source dirname for target by default
		DESTDIR="$(basename -- "$DUMPDIR")"
		[ -n "$2" ] && DESTDIR="$(realpath -m "$2")"
		extract_dump
	;;
	compact)
		DUMPDIR="$(realpath -e "$1")" || exit
		[ -d "$DUMPDIR" ] || fatal "$DUMPDIR is not a dir"
		exec 9>"$DUMPDIR/.lock" || fatal "Can't create $DUMPDIR/.lock"
		flock -n 9 || fatal "Another eterpack is updating $DUMPDIR"
		compact_dump
	;;
	test|check)
		DUMPDIR="$(realpath -e "$1")" || exit
		[ -d "$DUMPDIR" ] || fatal "$DUMPDIR is not a dir"
		test_dump
	;;
	info)
		DUMPDIR="$(realpath -e "$1")" || exit
		[ -d "$DUMPDIR" ] || fatal "$DUMPDIR is not a dir"
		print_info
	;;
	status)
		DUMPDIR="$(realpath -e "$1")" || exit
		[ -d "$DUMPDIR" ] || fatal "$DUMPDIR is not a dir"
		print_status
	;;
	-h|--help)
		echo $DESCR
		echo "Run with $0 command [options] args"

		echo
		echo "Create/Update backup:"
		echo "	$ eterpack update /path/from /path/to"

		#echo
		#echo "Repack backup (refresh zpaq files for minimize):"
		#echo "	$ eterbackup repack /path/from /path/to"

		echo
		echo "Restore backup:"
		echo "	$ eterpack restore /path/backup /path/to"

		echo
		echo "Compare backup with local files:"
		echo "	$ eterpack compare /path/backup /path/to"

		echo
		echo "Test backup integrity:"
		echo "	$ eterpack test /path/backup"

		echo
		echo "Remove empty parts left by updates without changes (not for encrypted backups):"
		echo "	$ eterpack compact /path/backup"

		echo
		echo "Print archives with their versions and sizes (--all lists every version):"
		echo "	$ eterpack info [--all] /path/backup"

		echo
		echo "Print backup status:"
		echo "	$ eterpack status /path/backup"

		echo
		echo "Options:"
		echo "	--depth N          - set depth for subdirs (1 by default) (update only)"
		echo "	--exclude name     - exclude dir 'name' from packing (full path or level dir name), can be repeated"
		echo "	--exclude-from file - read --exclude values from file (one per line, # starts a comment)"
		echo "	--execute command  - execute 'command' after every archive"
		echo "	--checksum         - force checking file contains, not date only"
		echo "	--password         - use password for encryption/decryption"
		echo "	--noattribute      - do not extra save owner/group, permissions and special files"
		echo "	--singlearchive    - pack all subdirs to one package"
		echo "	--followlinks      - follow links for primary dirs"
		echo "	--rollback N       - (for restore) roll back archive to N'th update or -N from end."
		echo "	--rollback 2019-02-18 - roll back to the date (UT, default time: 23:59:59)."
	;;
	*)
		echo "$DESCR" >&2
		echo "Run with -h or --help for help" >&2
		exit 1
	;;
esac

