feat: add motion support

Author: kylehuntsman

config.json

@@ -5,7 +5,7 @@
     "metricsListenOn": [],
     "replicationGroupId": "RG001",
     "restEndpoints": {
-        "localhost": "us-east-1",
+        "localhost": "location-motion-v1",
         "127.0.0.1": "us-east-1",
         "cloudserver-front": "us-east-1",
         "s3.docker.test": "us-east-1",
@@ -60,7 +60,7 @@
     },
     "clusters": 1,
     "log": {
-        "logLevel": "info",
+        "logLevel": "debug",
         "dumpLevel": "error"
     },
     "healthChecks": {
@@ -90,6 +90,10 @@
         "bindAddress": "localhost",
         "port": 9992
     },
+    "motionDaemon": {
+        "bindAddress": "localhost",
+        "port": 40080
+    },
     "recordLog": {
         "enabled": true,
         "recordLogName": "s3-recordlog"

constants.js

@@ -129,7 +129,7 @@ const constants = {
         },
     },
     /* eslint-disable camelcase */
-    externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true, dmf: true, azure_archive: true },
+    externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true, dmf: true, azure_archive: true, motion: true },
     // some of the available data backends  (if called directly rather
     // than through the multiple backend gateway) need a key provided
     // as a string as first parameter of the get/delete methods.
@@ -138,7 +138,7 @@ const constants = {
     // for external backends, don't call unless at least 1 minute
     // (60,000 milliseconds) since last call
     externalBackendHealthCheckInterval: 60000,
-    versioningNotImplBackends: { azure: true, gcp: true },
+    versioningNotImplBackends: { azure: true, gcp: true, motion: true },
     mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
     skipBatchDeleteBackends: { azure: true, gcp: true },
     s3HandledBackends: { azure: true, gcp: true },

lib/Config.js

@@ -1851,6 +1851,11 @@ class Config extends EventEmitter {
         this.locationConstraints[locationConstraint].details.pfsDaemonEndpoint;
     }
 
+    getMotionDaemonEndpoint(locationConstraint) {
+        return process.env[`${locationConstraint}_MOTION_ENDPOINT`] ||
+        this.locationConstraints[locationConstraint].details.motionDaemonEndpoint;
+    }
+
     isSameAzureAccount(locationConstraintSrc, locationConstraintDest) {
         if (!locationConstraintDest) {
             return true;

lib/data/external/motionClient.js

@@ -0,0 +1,24 @@
+import arsenal from 'arsenal';
+
+export class MotionClient {
+  constructor(config) {
+    this.motionEndpoint = config.motionEndpoint;
+    this.client = new arsenal.network.rest.RESTClient({
+      host: config.host,
+      port: config.port,
+      isPassthrough: true,
+    })
+  }
+
+  createMotionKey(bucketName, objectKey) {
+    return `${bucketName}/${objectKey}`;
+  }
+
+  put(stream, size, keyContext, reqUids, callback) {
+    const motionKey = this.createMotionKey(keyContext.bucketName, keyContext.objectKey);
+  }
+
+  get(keyContext, reqUids, callback) {
+    const motionKey = this.createMotionKey(keyContext.bucketName, keyContext.objectKey);
+  }
+}

locationConfig.json

@@ -108,5 +108,18 @@
         "legacyAwsBehavior": false,
         "isCold": true,
         "details": {}
+    },
+    "location-motion-v1": {
+        "type": "motion",
+        "objectId": "location-motion-v1",
+        "legacyAwsBehavior": false,
+        "isCold": false,
+        "details": {
+            "motionDaemonEndpoint": {
+                "host": "localhost",
+                "path": "/v0/blob",
+                "port": 40080
+            }
+        }
     }
 }

motionserver.js

@@ -0,0 +1,29 @@
+'use strict'; // eslint-disable-line strict
+
+const arsenal = require('arsenal');
+const { config } = require('./lib/Config.js');
+const logger = require('./lib/utilities/logger.js');
+
+const pfsServer = new arsenal.network.rest.RESTServer({
+    bindAddress: config.motionDaemon.bindAddress,
+    port: config.motionDaemon.port,
+    dataStore: new arsenal.storage.data.file.DataFileStore({
+        dataPath: config.motionDaemon.dataPath,
+        log: config.log,
+        noSync: config.motionDaemon.noSync,
+        noCache: config.motionDaemon.noCache,
+        isPassthrough: true,
+        isReadOnly: config.motionDaemon.isReadOnly,
+    }),
+    log: config.log,
+});
+
+motionServer.setup(err => {
+    if (err) {
+        logger.error('Error initializing REST MotionServer', {
+            error: err,
+        });
+        return;
+    }
+    pfsServer.start();
+});

package.json

@@ -21,7 +21,7 @@
   "dependencies": {
     "@azure/storage-blob": "^12.12.0",
     "@hapi/joi": "^17.1.0",
-    "arsenal": "git+https://github.com/scality/arsenal#8.1.105",
+    "arsenal": "git+https://github.com/filecoin-project/motion-arsenal",
     "async": "~2.5.0",
     "aws-sdk": "2.905.0",
     "bucketclient": "scality/bucketclient#8.1.9",
@@ -100,6 +100,7 @@
     "start_mdserver": "node mdserver.js",
     "start_dataserver": "node dataserver.js",
     "start_pfsserver": "node pfsserver.js",
+    "start_motionserver": "node motionserver.js",
     "start_s3server": "node index.js",
     "start_dmd": "npm-run-all --parallel start_mdserver start_dataserver",
     "start_utapi": "node lib/utapi/utapi.js",

yarn.lock

@@ -747,9 +747,10 @@ arraybuffer.slice@~0.0.7:
   resolved "https://registry.yarnpkg.com/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz#3bbc4275dd584cc1b10809b89d4e8b63a69e7675"
   integrity sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==
 
-"arsenal@git+https://github.com/scality/Arsenal#8.1.87":
-  version "8.1.87"
-  resolved "git+https://github.com/scality/Arsenal#ab0324da059c62171da4b9cf496dd067e22caac9"
+"arsenal@git+https://github.com/filecoin-project/motion-arsenal":
+  version "8.1.105"
+  uid "6e08c4fa9bddb7ac5733d69d5f1ee0cc3fb7bef5"
+  resolved "git+https://github.com/filecoin-project/motion-arsenal#6e08c4fa9bddb7ac5733d69d5f1ee0cc3fb7bef5"
   dependencies:
     "@azure/identity" "^3.1.1"
     "@azure/storage-blob" "^12.12.0"
@@ -766,7 +767,7 @@ arraybuffer.slice@~0.0.7:
     bson "4.0.0"
     debug "~4.1.0"
     diskusage "^1.1.1"
-    fcntl "github:scality/node-fcntl#0.2.0"
+    fcntl "github:scality/node-fcntl#0.2.2"
     hdclient scality/hdclient#1.1.5
     httpagent scality/httpagent#1.0.6
     https-proxy-agent "^2.2.0"
@@ -775,23 +776,23 @@ arraybuffer.slice@~0.0.7:
     joi "^17.6.0"
     level "~5.0.1"
     level-sublevel "~6.6.5"
-    mongodb "^3.0.1"
+    mongodb "^5.2.0"
     node-forge "^1.3.0"
     prom-client "14.2.0"
     simple-glob "^0.2.0"
-    socket.io "2.4.1"
-    socket.io-client "2.4.0"
-    sproxydclient scality/sproxydclient#8.0.7
+    socket.io "~4.6.1"
+    socket.io-client "~4.6.1"
+    sproxydclient "git+https://github.com/scality/sproxydclient#8.0.9"
     utf8 "3.0.0"
     uuid "^3.0.1"
     werelogs scality/werelogs#8.1.2
     xml2js "~0.4.23"
   optionalDependencies:
     ioctl "^2.0.2"
 
-"arsenal@git+https://github.com/scality/arsenal#8.1.105":
+"arsenal@git+https://github.com/kylehuntsman/Arsenal#6e08c4fa":
   version "8.1.105"
-  resolved "git+https://github.com/scality/arsenal#7c4f4611966fc21e2d79d74928a3ab0cea287a93"
+  resolved "git+https://github.com/kylehuntsman/Arsenal#6e08c4fa9bddb7ac5733d69d5f1ee0cc3fb7bef5"
   dependencies:
     "@azure/identity" "^3.1.1"
     "@azure/storage-blob" "^12.12.0"
@@ -831,6 +832,48 @@ arraybuffer.slice@~0.0.7:
   optionalDependencies:
     ioctl "^2.0.2"
 
+"arsenal@git+https://github.com/scality/Arsenal#8.1.87":
+  version "8.1.87"
+  resolved "git+https://github.com/scality/Arsenal#ab0324da059c62171da4b9cf496dd067e22caac9"
+  dependencies:
+    "@azure/identity" "^3.1.1"
+    "@azure/storage-blob" "^12.12.0"
+    "@types/async" "^3.2.12"
+    "@types/utf8" "^3.0.1"
+    JSONStream "^1.0.0"
+    agentkeepalive "^4.1.3"
+    ajv "6.12.3"
+    async "~2.6.4"
+    aws-sdk "^2.1005.0"
+    backo "^1.1.0"
+    base-x "3.0.8"
+    base62 "2.0.1"
+    bson "4.0.0"
+    debug "~4.1.0"
+    diskusage "^1.1.1"
+    fcntl "github:scality/node-fcntl#0.2.0"
+    hdclient scality/hdclient#1.1.5
+    httpagent scality/httpagent#1.0.6
+    https-proxy-agent "^2.2.0"
+    ioredis "^4.28.5"
+    ipaddr.js "1.9.1"
+    joi "^17.6.0"
+    level "~5.0.1"
+    level-sublevel "~6.6.5"
+    mongodb "^3.0.1"
+    node-forge "^1.3.0"
+    prom-client "14.2.0"
+    simple-glob "^0.2.0"
+    socket.io "2.4.1"
+    socket.io-client "2.4.0"
+    sproxydclient scality/sproxydclient#8.0.7
+    utf8 "3.0.0"
+    uuid "^3.0.1"
+    werelogs scality/werelogs#8.1.2
+    xml2js "~0.4.23"
+  optionalDependencies:
+    ioctl "^2.0.2"
+
 asn1@~0.2.3:
   version "0.2.6"
   resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.6.tgz#0d3a7bb6e64e02a90c0303b31f292868ea09a08d"