added all my scriptssynced/master

1 files changed, 283 insertions, 0 deletions

diff --git a/git-annex-remote-rclone b/git-annex-remote-rclone
new file mode 100755
index 0000000..53e342b
--- /dev/null
+++ b/git-annex-remote-rclone
@@ -0,0 +1,283 @@
+#!/bin/bash
+# emacs: -*- mode: python; tab-width: 4; indent-tabs-mode: t -*-
+# ex: set sts=4 ts=4 sw=4 noet:
+#
+# git-annex-remote-rclone - wrapper to enable use of rclone-supported cloud providers as git-annex special remotes.
+#
+# Install in PATH as git-annex-remote-rclone
+#
+# Copyright (C) 2016-2022  Daniel Dent
+#               2022       git-annex-remote-rclone contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of version 3 of the GNU
+# General Public License as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# Based on work originally copyright 2013 Joey Hess which was licenced under the GNU GPL version 3 or higher.
+#
+
+set -e
+
+# This program speaks a line-based protocol on stdin and stdout.
+# When running any commands, their stdout should be redirected to stderr
+# (or /dev/null) to avoid messing up the protocol.
+runcmd () {
+	"$@" >&2
+}
+
+# Gets a value from the remote's configuration, and stores it in RET
+getconfig () {
+	ask GETCONFIG "$1"
+}
+
+# Stores a value in the remote's configuration.
+setconfig () {
+	echo SETCONFIG "$1" "$2"
+}
+
+validate_layout() {
+	if [ -z "$RCLONE_LAYOUT" ]; then
+		RCLONE_LAYOUT="lower"
+	fi
+	case "$RCLONE_LAYOUT" in
+		lower|directory|nodir|mixed|frankencase)
+			;;
+		*)
+			echo "INITREMOTE-FAILURE rclone_layout setting not recognized"
+			exit 1
+			;;
+	esac
+}
+
+# Sets LOC to the location to use to store a key.
+calclocation () {
+	case "$RCLONE_LAYOUT" in
+		lower)
+			ask DIRHASH-LOWER "$1"
+			LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$RET"
+			;;
+		directory)
+			ask DIRHASH-LOWER "$1"
+			LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$RET$1/"
+			;;
+		nodir)
+			LOC="$REMOTE_TARGET:$REMOTE_PREFIX/"
+			;;
+		mixed)
+			ask DIRHASH "$1"
+			LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$RET"
+			;;
+		frankencase)
+			ask DIRHASH "$1"
+			lret=$(echo "$RET" | tr '[:upper:]' '[:lower:]')
+			LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$lret"
+			;;
+	esac
+}
+
+# Asks for some value, and stores it in RET
+ask () {
+	echo "$1" "$2"
+	read -r resp
+	# Strip trailing carriage return, if present
+	resp="${resp%$'\r'}"
+	if echo "$resp" | grep '^VALUE '>/dev/null; then
+		RET=$(echo "$resp" | cut -f2- -d' ')
+		else
+		RET=""
+	fi
+}
+
+printcmdresult() {
+	cmd="$1"
+	rc="$2"
+	out="$3"
+	# Replace explicit newline since we must provide 1 line DEBUG
+	# Fancy sed is from Example 5 of https://linuxhint.com/newline_replace_sed
+	# which worked on Linux and OSX.
+	# shellcheck disable=SC2016
+	out_safe=$(echo "$out" | sed -ne 'H;${x;s/\n/\\n/g;s/^,//;p;}')
+	echo "DEBUG '$cmd' exited with rc=$rc and stdout=${out_safe}"
+}
+
+GREP () {
+	set +e
+	out=$(grep "$@")
+	rc=$?
+	set -e
+	printcmdresult "grep \"$*\"" "$rc" "$out"
+	return $rc
+}
+
+do_checkpresent() {
+	key="$1"
+	dest="$2"
+
+	res=0
+	check_result=$(rclone size --json "$dest" 2>/dev/null) || res=$?
+	printcmdresult "rclone size --json \"$dest\"" "$res" "$check_result"
+	count=$(echo "$check_result" | sed -En 's/^.*"count":\s*([0-9]+).*$/\1/ p')
+	if [[ $res -eq 0 && "$count" -ge 1 ]]; then
+		# Any nonzero object count means present.
+		# Some rclone backends support multiple
+		# files under one name.
+		echo CHECKPRESENT-SUCCESS "$key"
+	elif [[ $res -eq 0 && -n "$count" && "$count" -eq 0 ]]; then
+		# A 0 object count means an empty directory.
+		echo CHECKPRESENT-FAILURE "$key"
+	elif [[ $res -eq 3 || $res -eq 4 ]]; then
+		# file or directory doesn't exist
+		# see https://rclone.org/docs/#exit-code
+		echo CHECKPRESENT-FAILURE "$key"
+	else
+		echo CHECKPRESENT-UNKNOWN "$key" "remote currently unavailable or git-annex-remote-rclone failed to parse rclone output"
+	fi
+}
+
+do_remove() {
+	key="$1"
+	dest="$2"
+
+	# Note that it's not a failure to remove a
+	# key that is not present.
+	if remove_result=$(rclone delete --retries 1 "$dest" 2>&1); then
+		echo REMOVE-SUCCESS "$key"
+	else
+		if echo "$remove_result" | GREP ' directory not found'; then
+			echo REMOVE-SUCCESS "$key"
+		else
+			echo REMOVE-FAILURE "$key"
+		fi
+	fi
+}
+
+# This has to come first, to get the protocol started.
+echo VERSION 1
+
+while read -r line; do
+	# Strip trailing carriage return, if present
+	line="${line%$'\r'}"
+	# shellcheck disable=SC2086
+	set -- $line
+	case "$1" in
+		INITREMOTE)
+			# Do anything necessary to create resources
+			# used by the remote. Try to be idempotent.
+			#
+			# Use GETCONFIG to get any needed configuration
+			# settings, and SETCONFIG to set any persistent
+			# configuration settings.
+			#
+			# (Note that this is not run every time, only when
+			# git annex initremote or git annex enableremote is
+			# run.)
+
+			getconfig prefix
+			REMOTE_PREFIX=$RET
+			if [ -z "$REMOTE_PREFIX" ]; then
+				REMOTE_PREFIX="git-annex"
+			fi
+			if [ "$REMOTE_PREFIX" == "/" ]; then
+				echo INITREMOTE-FAILURE "storing objects directly in the root (/) is not supported"
+			fi
+			setconfig prefix $REMOTE_PREFIX
+
+			getconfig target
+			REMOTE_TARGET=$RET
+			setconfig target "$REMOTE_TARGET"
+
+			getconfig rclone_layout
+			RCLONE_LAYOUT=$RET
+			validate_layout
+			setconfig rclone_layout "$RCLONE_LAYOUT"
+
+			if [ -z "$REMOTE_TARGET" ]; then
+				echo INITREMOTE-FAILURE "rclone remote target must be specified (use target= parameter)"
+			fi
+
+			if runcmd rclone mkdir "$REMOTE_TARGET:$REMOTE_PREFIX"; then
+				echo INITREMOTE-SUCCESS
+			else
+				echo INITREMOTE-FAILURE "Failed to create directory on remote. Ensure that 'rclone config' has been run."
+			fi
+		;;
+		PREPARE)
+			# Use GETCONFIG to get configuration settings,
+			# and do anything needed to get ready for using the
+			# special remote here.
+
+			getconfig prefix
+			REMOTE_PREFIX="$RET"
+
+			getconfig target
+			REMOTE_TARGET="$RET"
+
+			getconfig rclone_layout
+			RCLONE_LAYOUT="$RET"
+			validate_layout
+
+			echo PREPARE-SUCCESS
+		;;
+		TRANSFER)
+			op="$2"
+			key="$3"
+			shift 3
+			file="$*"
+			case "$op" in
+				STORE)
+					# Store the file to a location
+					# based on the key.
+					# XXX when at all possible, send PROGRESS
+					calclocation "$key"
+					if [ ! -e "$file" ]; then
+						echo TRANSFER-FAILURE STORE "$key" "asked to store non-existent file $file"
+					else
+						if runcmd rclone copy "$file" "$LOC"; then
+							echo TRANSFER-SUCCESS STORE "$key"
+						else
+							echo TRANSFER-FAILURE STORE "$key"
+						fi
+					fi
+				;;
+				RETRIEVE)
+					# Retrieve from a location based on
+					# the key, outputting to the file.
+					# XXX when easy to do, send PROGRESS
+					calclocation "$key"
+					# http://stackoverflow.com/questions/31396985/why-is-mktemp-on-os-x-broken-with-a-command-that-worked-on-linux
+					if GA_RC_TEMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/rclone-annex-tmp.XXXXXXXXX") &&
+						runcmd rclone copy "$LOC$key" "$GA_RC_TEMP_DIR" &&
+						mv "$GA_RC_TEMP_DIR/$key" "$file" &&
+						rmdir "$GA_RC_TEMP_DIR"; then
+						echo TRANSFER-SUCCESS RETRIEVE "$key"
+					else
+						echo TRANSFER-FAILURE RETRIEVE "$key"
+					fi
+				;;
+			esac
+		;;
+		CHECKPRESENT)
+			key="$2"
+			calclocation "$key"
+
+			do_checkpresent "$key" "$LOC$key"
+		;;
+		REMOVE)
+			key="$2"
+			calclocation "$key"
+
+			do_remove "$key" "$LOC$key"
+		;;
+		*)
+			# The requests listed above are all the ones
+			# that are required to be supported, so it's fine
+			# to say that any other request is unsupported.
+			echo UNSUPPORTED-REQUEST
+		;;
+	esac
+done
+
+
+# XXX anything that needs to be done at shutdown can be done here