path: root/git-annex-remote-rclone

#!/bin/bash
# emacs: -*- mode: python; tab-width: 4; indent-tabs-mode: t -*-
# ex: set sts=4 ts=4 sw=4 noet:
#
# git-annex-remote-rclone - wrapper to enable use of rclone-supported cloud providers as git-annex special remotes.
#
# Install in PATH as git-annex-remote-rclone
#
# Copyright (C) 2016-2022  Daniel Dent
#               2022       git-annex-remote-rclone contributors
#
# This program is free software: you can redistribute it and/or modify it under the terms of version 3 of the GNU
# General Public License as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# Based on work originally copyright 2013 Joey Hess which was licenced under the GNU GPL version 3 or higher.
#

set -e

# This program speaks a line-based protocol on stdin and stdout.
# When running any commands, their stdout should be redirected to stderr
# (or /dev/null) to avoid messing up the protocol.
runcmd () {
	"$@" >&2
}

# Gets a value from the remote's configuration, and stores it in RET
getconfig () {
	ask GETCONFIG "$1"
}

# Stores a value in the remote's configuration.
setconfig () {
	echo SETCONFIG "$1" "$2"
}

validate_layout() {
	if [ -z "$RCLONE_LAYOUT" ]; then
		RCLONE_LAYOUT="lower"
	fi
	case "$RCLONE_LAYOUT" in
		lower|directory|nodir|mixed|frankencase)
			;;
		*)
			echo "INITREMOTE-FAILURE rclone_layout setting not recognized"
			exit 1
			;;
	esac
}

# Sets LOC to the location to use to store a key.
calclocation () {
	case "$RCLONE_LAYOUT" in
		lower)
			ask DIRHASH-LOWER "$1"
			LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$RET"
			;;
		directory)
			ask DIRHASH-LOWER "$1"
			LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$RET$1/"
			;;
		nodir)
			LOC="$REMOTE_TARGET:$REMOTE_PREFIX/"
			;;
		mixed)
			ask DIRHASH "$1"
			LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$RET"
			;;
		frankencase)
			ask DIRHASH "$1"
			lret=$(echo "$RET" | tr '[:upper:]' '[:lower:]')
			LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$lret"
			;;
	esac
}

# Asks for some value, and stores it in RET
ask () {
	echo "$1" "$2"
	read -r resp
	# Strip trailing carriage return, if present
	resp="${resp%$'\r'}"
	if echo "$resp" | grep '^VALUE '>/dev/null; then
		RET=$(echo "$resp" | cut -f2- -d' ')
		else
		RET=""
	fi
}

printcmdresult() {
	cmd="$1"
	rc="$2"
	out="$3"
	# Replace explicit newline since we must provide 1 line DEBUG
	# Fancy sed is from Example 5 of https://linuxhint.com/newline_replace_sed
	# which worked on Linux and OSX.
	# shellcheck disable=SC2016
	out_safe=$(echo "$out" | sed -ne 'H;${x;s/\n/\\n/g;s/^,//;p;}')
	echo "DEBUG '$cmd' exited with rc=$rc and stdout=${out_safe}"
}

GREP () {
	set +e
	out=$(grep "$@")
	rc=$?
	set -e
	printcmdresult "grep \"$*\"" "$rc" "$out"
	return $rc
}

do_checkpresent() {
	key="$1"
	dest="$2"

	res=0
	check_result=$(rclone size --json "$dest" 2>/dev/null) || res=$?
	printcmdresult "rclone size --json \"$dest\"" "$res" "$check_result"
	count=$(echo "$check_result" | sed -En 's/^.*"count":\s*([0-9]+).*$/\1/ p')
	if [[ $res -eq 0 && "$count" -ge 1 ]]; then
		# Any nonzero object count means present.
		# Some rclone backends support multiple
		# files under one name.
		echo CHECKPRESENT-SUCCESS "$key"
	elif [[ $res -eq 0 && -n "$count" && "$count" -eq 0 ]]; then
		# A 0 object count means an empty directory.
		echo CHECKPRESENT-FAILURE "$key"
	elif [[ $res -eq 3 || $res -eq 4 ]]; then
		# file or directory doesn't exist
		# see https://rclone.org/docs/#exit-code
		echo CHECKPRESENT-FAILURE "$key"
	else
		echo CHECKPRESENT-UNKNOWN "$key" "remote currently unavailable or git-annex-remote-rclone failed to parse rclone output"
	fi
}

do_remove() {
	key="$1"
	dest="$2"

	# Note that it's not a failure to remove a
	# key that is not present.
	if remove_result=$(rclone delete --retries 1 "$dest" 2>&1); then
		echo REMOVE-SUCCESS "$key"
	else
		if echo "$remove_result" | GREP ' directory not found'; then
			echo REMOVE-SUCCESS "$key"
		else
			echo REMOVE-FAILURE "$key"
		fi
	fi
}

# This has to come first, to get the protocol started.
echo VERSION 1

while read -r line; do
	# Strip trailing carriage return, if present
	line="${line%$'\r'}"
	# shellcheck disable=SC2086
	set -- $line
	case "$1" in
		INITREMOTE)
			# Do anything necessary to create resources
			# used by the remote. Try to be idempotent.
			#
			# Use GETCONFIG to get any needed configuration
			# settings, and SETCONFIG to set any persistent
			# configuration settings.
			#
			# (Note that this is not run every time, only when
			# git annex initremote or git annex enableremote is
			# run.)

			getconfig prefix
			REMOTE_PREFIX=$RET
			if [ -z "$REMOTE_PREFIX" ]; then
				REMOTE_PREFIX="git-annex"
			fi
			if [ "$REMOTE_PREFIX" == "/" ]; then
				echo INITREMOTE-FAILURE "storing objects directly in the root (/) is not supported"
			fi
			setconfig prefix $REMOTE_PREFIX

			getconfig target
			REMOTE_TARGET=$RET
			setconfig target "$REMOTE_TARGET"

			getconfig rclone_layout
			RCLONE_LAYOUT=$RET
			validate_layout
			setconfig rclone_layout "$RCLONE_LAYOUT"

			if [ -z "$REMOTE_TARGET" ]; then
				echo INITREMOTE-FAILURE "rclone remote target must be specified (use target= parameter)"
			fi

			if runcmd rclone mkdir "$REMOTE_TARGET:$REMOTE_PREFIX"; then
				echo INITREMOTE-SUCCESS
			else
				echo INITREMOTE-FAILURE "Failed to create directory on remote. Ensure that 'rclone config' has been run."
			fi
		;;
		PREPARE)
			# Use GETCONFIG to get configuration settings,
			# and do anything needed to get ready for using the
			# special remote here.

			getconfig prefix
			REMOTE_PREFIX="$RET"

			getconfig target
			REMOTE_TARGET="$RET"

			getconfig rclone_layout
			RCLONE_LAYOUT="$RET"
			validate_layout

			echo PREPARE-SUCCESS
		;;
		TRANSFER)
			op="$2"
			key="$3"
			shift 3
			file="$*"
			case "$op" in
				STORE)
					# Store the file to a location
					# based on the key.
					# XXX when at all possible, send PROGRESS
					calclocation "$key"
					if [ ! -e "$file" ]; then
						echo TRANSFER-FAILURE STORE "$key" "asked to store non-existent file $file"
					else
						if runcmd rclone copy "$file" "$LOC"; then
							echo TRANSFER-SUCCESS STORE "$key"
						else
							echo TRANSFER-FAILURE STORE "$key"
						fi
					fi
				;;
				RETRIEVE)
					# Retrieve from a location based on
					# the key, outputting to the file.
					# XXX when easy to do, send PROGRESS
					calclocation "$key"
					# http://stackoverflow.com/questions/31396985/why-is-mktemp-on-os-x-broken-with-a-command-that-worked-on-linux
					if GA_RC_TEMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/rclone-annex-tmp.XXXXXXXXX") &&
						runcmd rclone copy "$LOC$key" "$GA_RC_TEMP_DIR" &&
						mv "$GA_RC_TEMP_DIR/$key" "$file" &&
						rmdir "$GA_RC_TEMP_DIR"; then
						echo TRANSFER-SUCCESS RETRIEVE "$key"
					else
						echo TRANSFER-FAILURE RETRIEVE "$key"
					fi
				;;
			esac
		;;
		CHECKPRESENT)
			key="$2"
			calclocation "$key"

			do_checkpresent "$key" "$LOC$key"
		;;
		REMOVE)
			key="$2"
			calclocation "$key"

			do_remove "$key" "$LOC$key"
		;;
		*)
			# The requests listed above are all the ones
			# that are required to be supported, so it's fine
			# to say that any other request is unsupported.
			echo UNSUPPORTED-REQUEST
		;;
	esac
done


# XXX anything that needs to be done at shutdown can be done here