summaryrefslogtreecommitdiff
path: root/git-annex-remote-rclone
diff options
context:
space:
mode:
Diffstat (limited to 'git-annex-remote-rclone')
-rwxr-xr-xgit-annex-remote-rclone283
1 files changed, 283 insertions, 0 deletions
diff --git a/git-annex-remote-rclone b/git-annex-remote-rclone
new file mode 100755
index 0000000..53e342b
--- /dev/null
+++ b/git-annex-remote-rclone
@@ -0,0 +1,283 @@
+#!/bin/bash
+# emacs: -*- mode: python; tab-width: 4; indent-tabs-mode: t -*-
+# ex: set sts=4 ts=4 sw=4 noet:
+#
+# git-annex-remote-rclone - wrapper to enable use of rclone-supported cloud providers as git-annex special remotes.
+#
+# Install in PATH as git-annex-remote-rclone
+#
+# Copyright (C) 2016-2022 Daniel Dent
+# 2022 git-annex-remote-rclone contributors
+#
+# This program is free software: you can redistribute it and/or modify it under the terms of version 3 of the GNU
+# General Public License as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# Based on work originally copyright 2013 Joey Hess which was licenced under the GNU GPL version 3 or higher.
+#
+
+set -e
+
+# This program speaks a line-based protocol on stdin and stdout.
+# When running any commands, their stdout should be redirected to stderr
+# (or /dev/null) to avoid messing up the protocol.
+runcmd () {
+ "$@" >&2
+}
+
+# Gets a value from the remote's configuration, and stores it in RET
+getconfig () {
+ ask GETCONFIG "$1"
+}
+
+# Stores a value in the remote's configuration.
+setconfig () {
+ echo SETCONFIG "$1" "$2"
+}
+
+validate_layout() {
+ if [ -z "$RCLONE_LAYOUT" ]; then
+ RCLONE_LAYOUT="lower"
+ fi
+ case "$RCLONE_LAYOUT" in
+ lower|directory|nodir|mixed|frankencase)
+ ;;
+ *)
+ echo "INITREMOTE-FAILURE rclone_layout setting not recognized"
+ exit 1
+ ;;
+ esac
+}
+
+# Sets LOC to the location to use to store a key.
+calclocation () {
+ case "$RCLONE_LAYOUT" in
+ lower)
+ ask DIRHASH-LOWER "$1"
+ LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$RET"
+ ;;
+ directory)
+ ask DIRHASH-LOWER "$1"
+ LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$RET$1/"
+ ;;
+ nodir)
+ LOC="$REMOTE_TARGET:$REMOTE_PREFIX/"
+ ;;
+ mixed)
+ ask DIRHASH "$1"
+ LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$RET"
+ ;;
+ frankencase)
+ ask DIRHASH "$1"
+ lret=$(echo "$RET" | tr '[:upper:]' '[:lower:]')
+ LOC="$REMOTE_TARGET:$REMOTE_PREFIX/$lret"
+ ;;
+ esac
+}
+
+# Asks for some value, and stores it in RET
+ask () {
+ echo "$1" "$2"
+ read -r resp
+ # Strip trailing carriage return, if present
+ resp="${resp%$'\r'}"
+ if echo "$resp" | grep '^VALUE '>/dev/null; then
+ RET=$(echo "$resp" | cut -f2- -d' ')
+ else
+ RET=""
+ fi
+}
+
+printcmdresult() {
+ cmd="$1"
+ rc="$2"
+ out="$3"
+ # Replace explicit newline since we must provide 1 line DEBUG
+ # Fancy sed is from Example 5 of https://linuxhint.com/newline_replace_sed
+ # which worked on Linux and OSX.
+ # shellcheck disable=SC2016
+ out_safe=$(echo "$out" | sed -ne 'H;${x;s/\n/\\n/g;s/^,//;p;}')
+ echo "DEBUG '$cmd' exited with rc=$rc and stdout=${out_safe}"
+}
+
+GREP () {
+ set +e
+ out=$(grep "$@")
+ rc=$?
+ set -e
+ printcmdresult "grep \"$*\"" "$rc" "$out"
+ return $rc
+}
+
+do_checkpresent() {
+ key="$1"
+ dest="$2"
+
+ res=0
+ check_result=$(rclone size --json "$dest" 2>/dev/null) || res=$?
+ printcmdresult "rclone size --json \"$dest\"" "$res" "$check_result"
+ count=$(echo "$check_result" | sed -En 's/^.*"count":\s*([0-9]+).*$/\1/ p')
+ if [[ $res -eq 0 && "$count" -ge 1 ]]; then
+ # Any nonzero object count means present.
+ # Some rclone backends support multiple
+ # files under one name.
+ echo CHECKPRESENT-SUCCESS "$key"
+ elif [[ $res -eq 0 && -n "$count" && "$count" -eq 0 ]]; then
+ # A 0 object count means an empty directory.
+ echo CHECKPRESENT-FAILURE "$key"
+ elif [[ $res -eq 3 || $res -eq 4 ]]; then
+ # file or directory doesn't exist
+ # see https://rclone.org/docs/#exit-code
+ echo CHECKPRESENT-FAILURE "$key"
+ else
+ echo CHECKPRESENT-UNKNOWN "$key" "remote currently unavailable or git-annex-remote-rclone failed to parse rclone output"
+ fi
+}
+
+do_remove() {
+ key="$1"
+ dest="$2"
+
+ # Note that it's not a failure to remove a
+ # key that is not present.
+ if remove_result=$(rclone delete --retries 1 "$dest" 2>&1); then
+ echo REMOVE-SUCCESS "$key"
+ else
+ if echo "$remove_result" | GREP ' directory not found'; then
+ echo REMOVE-SUCCESS "$key"
+ else
+ echo REMOVE-FAILURE "$key"
+ fi
+ fi
+}
+
+# This has to come first, to get the protocol started.
+echo VERSION 1
+
+while read -r line; do
+ # Strip trailing carriage return, if present
+ line="${line%$'\r'}"
+ # shellcheck disable=SC2086
+ set -- $line
+ case "$1" in
+ INITREMOTE)
+ # Do anything necessary to create resources
+ # used by the remote. Try to be idempotent.
+ #
+ # Use GETCONFIG to get any needed configuration
+ # settings, and SETCONFIG to set any persistent
+ # configuration settings.
+ #
+ # (Note that this is not run every time, only when
+ # git annex initremote or git annex enableremote is
+ # run.)
+
+ getconfig prefix
+ REMOTE_PREFIX=$RET
+ if [ -z "$REMOTE_PREFIX" ]; then
+ REMOTE_PREFIX="git-annex"
+ fi
+ if [ "$REMOTE_PREFIX" == "/" ]; then
+ echo INITREMOTE-FAILURE "storing objects directly in the root (/) is not supported"
+ fi
+ setconfig prefix $REMOTE_PREFIX
+
+ getconfig target
+ REMOTE_TARGET=$RET
+ setconfig target "$REMOTE_TARGET"
+
+ getconfig rclone_layout
+ RCLONE_LAYOUT=$RET
+ validate_layout
+ setconfig rclone_layout "$RCLONE_LAYOUT"
+
+ if [ -z "$REMOTE_TARGET" ]; then
+ echo INITREMOTE-FAILURE "rclone remote target must be specified (use target= parameter)"
+ fi
+
+ if runcmd rclone mkdir "$REMOTE_TARGET:$REMOTE_PREFIX"; then
+ echo INITREMOTE-SUCCESS
+ else
+ echo INITREMOTE-FAILURE "Failed to create directory on remote. Ensure that 'rclone config' has been run."
+ fi
+ ;;
+ PREPARE)
+ # Use GETCONFIG to get configuration settings,
+ # and do anything needed to get ready for using the
+ # special remote here.
+
+ getconfig prefix
+ REMOTE_PREFIX="$RET"
+
+ getconfig target
+ REMOTE_TARGET="$RET"
+
+ getconfig rclone_layout
+ RCLONE_LAYOUT="$RET"
+ validate_layout
+
+ echo PREPARE-SUCCESS
+ ;;
+ TRANSFER)
+ op="$2"
+ key="$3"
+ shift 3
+ file="$*"
+ case "$op" in
+ STORE)
+ # Store the file to a location
+ # based on the key.
+ # XXX when at all possible, send PROGRESS
+ calclocation "$key"
+ if [ ! -e "$file" ]; then
+ echo TRANSFER-FAILURE STORE "$key" "asked to store non-existent file $file"
+ else
+ if runcmd rclone copy "$file" "$LOC"; then
+ echo TRANSFER-SUCCESS STORE "$key"
+ else
+ echo TRANSFER-FAILURE STORE "$key"
+ fi
+ fi
+ ;;
+ RETRIEVE)
+ # Retrieve from a location based on
+ # the key, outputting to the file.
+ # XXX when easy to do, send PROGRESS
+ calclocation "$key"
+ # http://stackoverflow.com/questions/31396985/why-is-mktemp-on-os-x-broken-with-a-command-that-worked-on-linux
+ if GA_RC_TEMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/rclone-annex-tmp.XXXXXXXXX") &&
+ runcmd rclone copy "$LOC$key" "$GA_RC_TEMP_DIR" &&
+ mv "$GA_RC_TEMP_DIR/$key" "$file" &&
+ rmdir "$GA_RC_TEMP_DIR"; then
+ echo TRANSFER-SUCCESS RETRIEVE "$key"
+ else
+ echo TRANSFER-FAILURE RETRIEVE "$key"
+ fi
+ ;;
+ esac
+ ;;
+ CHECKPRESENT)
+ key="$2"
+ calclocation "$key"
+
+ do_checkpresent "$key" "$LOC$key"
+ ;;
+ REMOVE)
+ key="$2"
+ calclocation "$key"
+
+ do_remove "$key" "$LOC$key"
+ ;;
+ *)
+ # The requests listed above are all the ones
+ # that are required to be supported, so it's fine
+ # to say that any other request is unsupported.
+ echo UNSUPPORTED-REQUEST
+ ;;
+ esac
+done
+
+
+# XXX anything that needs to be done at shutdown can be done here
generated by cgit v1.2.3-70-g09d2 (git 2.47.0) at 2024-11-09 01:38:30 +0000