File Transfer

File Transfer

storage_azure

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_azure"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "az_acc_name": "{vtk-test-azure-account}",
        "az_acc_key": "{vtk-test-azure-account-access-key}",
        "az_container": "hls",
        "files": [
            "*"
        ]
      }
    },
    {
      "tool": "storage:put",
      "parameters": {
        "az_acc_name": "{vtk-test-azure-account}",
        "az_sas_token": "{vtk-test-azure-sas-token}",
        "az_container": "12345",
        "overwrite": "True",
        "files": [
            "hls/*"
        ]
      }
    },
    {
      "tool": "storage:put",
      "parameters": {
        "az_acc_name": "{vtk-test-azure-account}",
        "az_acc_key": "{vtk-test-azure-account-access-key}",
        "az_container": "12345",
        "overwrite": "True",
        "files": [
            "hls/*"
        ]
      }
    }
  ]
}

storage_b2

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_get"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://{vtk-test-b2-account-new}:{vtk-test-b2-secret-new}@s3.us-east-005.backblazeb2.com/vtk-test-bucket/vtk-i-test/",
        "files": [
          "test-upload.txt"
        ]
      }
    },
    {
      "tool": "storage:put",
      "parameters": {
        "location": "s3://{vtk-test-b2-account-new}:{vtk-test-b2-secret-new}@s3.us-east-005.backblazeb2.com/vtk-test-bucket/vtk-i-test/OUT/",
        "files": [
          "test-upload.txt"
        ],
        "overwrite": "true"
      }
    }
  ]
}

storage_check

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_check"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:check",
      "parameters": {
        "location": "s3://{com-castlabs-vtk-itest-access-keys}@com.castlabs.vtk.itest/IN",
        "files": [
          "tos_5s_video_1920x1080_2mbps.mp4"
        ],
        "mode": "COMPLETE_ON_PRESENCE"
      }
    },
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://{com-castlabs-vtk-itest-access-keys}@com.castlabs.vtk.itest/IN",
        "files": [
          "notavailable.mp4"
        ]
      }
    }
  ]
}

storage_check_b2

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_check"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:check",
      "parameters": {
        "location": "s3://{vtk-test-b2-account-new}:{vtk-test-b2-secret-new}@s3.us-east-005.backblazeb2.com/vtk-test-bucket/vtk-i-test/",
        "files": [
          "test-upload.txt"
        ],
        "mode": "COMPLETE_ON_PRESENCE"
      }
    },
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://{vtk-test-b2-account-new}:{vtk-test-b2-secret-new}@s3.us-east-005.backblazeb2.com/vtk-test-bucket/vtk-i-test/",
        "files": [
          "notavailable.mp4"
        ]
      }
    }
  ]
}

storage_check_jobrole

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_check",
    "jobrole"
  ],
  "region": "aws:eu-west-1",
  "role_arn": "arn:aws:iam::685872506356:role/video-toolkit-integration-test-staging",
  "tasks": [
    {
      "tool": "storage:check",
      "parameters": {
        "location": "s3://com.castlabs.vtk.itest/IN",
        "files": [
          "tos_5s_video_1920x1080_2mbps.mp4"
        ],
        "mode": "COMPLETE_ON_PRESENCE"
      }
    },
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://{com-castlabs-vtk-itest-access-keys}@com.castlabs.vtk.itest/IN",
        "files": [
          "notavailable.mp4"
        ]
      }
    }
  ]
}

storage_check_toolrole

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_check",
    "tool_role"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:check",
      "parameters": {
        "role_arn": "arn:aws:iam::685872506356:role/video-toolkit-integration-test-staging",
        "location": "s3://com.castlabs.vtk.itest/IN",
        "files": [
          "tos_5s_video_1920x1080_2mbps.mp4"
        ],
        "mode": "COMPLETE_ON_PRESENCE"
      }
    },
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://{com-castlabs-vtk-itest-access-keys}@com.castlabs.vtk.itest/IN",
        "files": [
          "notavailable.mp4"
        ]
      }
    }
  ]
}

storage_get

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_get"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://{com-castlabs-vtk-itest-access-keys}@com.castlabs.vtk.itest/",
        "files": [
          "integration-test-storage/*"
        ]
      }
    }
  ]
}

storage_get_basedir

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_get_basedir"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://{com-castlabs-vtk-itest-access-keys}@com.castlabs.vtk.itest/integration-test-storage/",
        "base_dir": "test",
        "files": [
          "*"
        ]
      }
    }
  ]
}

storage_get_http

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_get_http"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "location": "https://demo.cf.castlabs.com/public/Transcodes/IN/PETV/",
        "files": [
          "PETVA.mp4"
        ]
      }
    }
  ]
}

storage_get_http_credentials

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_get"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://com.castlabs.vtk.itest/",
        "http_credential_source": "https://6aqr4f4rnh.execute-api.us-east-1.amazonaws.com/prod/not_relevant",
        "files": [
          "integration-test-storage/*"
        ]
      }
    }
  ]
}

storage_get_jobrole

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_get",
    "jobrole"
  ],
  "region": "aws:eu-west-1",
  "role_arn": "arn:aws:iam::685872506356:role/video-toolkit-integration-test-staging",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://com.castlabs.vtk.itest/",
        "files": [
          "integration-test-storage/*"
        ]
      }
    }
  ]
}

storage_get_toolrole

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_get",
    "tool_role"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "role_arn": "arn:aws:iam::685872506356:role/video-toolkit-integration-test-staging",
        "location": "s3://com.castlabs.vtk.itest/",
        "files": [
          "integration-test-storage/*"
        ]
      }
    }
  ]
}

storage_google

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_google"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "location": "gs://729d13e872b4a00ae5358afe57d8b7c7-hls/downloads/",
        "base_dir": "hereplease",
        "gc_adc_string": "{vtk-google-adc}",
        "files": [
            "*"
        ]
      }
    },
    {
      "tool": "storage:put",
      "parameters": {
        "location": "gs://d75dcf034db55da8303099057fc00207-hls/uploads/",
        "gc_adc_string": "{vtk-google-adc}",
        "overwrite": "True",
        "files": [
            "hereplease/*"
        ]
      }
    }
  ]
}

storage_put

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_put"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://{com-castlabs-vtk-itest-access-keys}@com.castlabs.vtk.itest",
        "files": [
          "integration-test-storage/*"
        ]
      }
    },
    {
      "tool": "storage:put",
      "parameters": {
        "location": "s3://{com-castlabs-vtk-itest-access-keys}@com.castlabs.vtk.itest/OUT/filetransfer-storage_put_{job_id}",
        "files": [
          "*.webm"
        ]
      }
    }
  ]
}

storage_put_basedir

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_put_basedir"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://{com-castlabs-vtk-itest-access-keys}@com.castlabs.vtk.itest",
        "base_dir" : "in",
        "files": [
          "integration-test-storage/*"
        ]
      }
    },
    {
      "tool": "storage:put",
      "parameters": {
        "location": "s3://{com-castlabs-vtk-itest-access-keys}@com.castlabs.vtk.itest/OUT/filetransfer-storage_put_{job_id}/out",
        "base_dir" : "in",
        "files": [
          "*.webm"
        ]
      }
    }
  ]
}

storage_put_http_credentials

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_put"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://com.castlabs.vtk.itest",
        "http_credential_source": "https://6aqr4f4rnh.execute-api.us-east-1.amazonaws.com/prod/not_relevant",
        "files": [
          "integration-test-storage/*"
        ]
      }
    },
    {
      "tool": "storage:put",
      "parameters": {
        "location": "s3://com.castlabs.vtk.itest/OUT/filetransfer-storage_put_{job_id}",
        "http_credential_source": "https://6aqr4f4rnh.execute-api.us-east-1.amazonaws.com/prod/not_relevant",
        "files": [
          "*.webm"
        ]
      }
    }
  ]
}

storage_put_jobrole

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_put",
    "jobrole"
  ],
  "region": "aws:eu-west-1",
  "role_arn": "arn:aws:iam::685872506356:role/video-toolkit-integration-test-staging",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://com.castlabs.vtk.itest",
        "files": [
          "integration-test-storage/*"
        ]
      }
    },
    {
      "tool": "storage:put",
      "parameters": {
        "location": "s3://com.castlabs.vtk.itest/OUT/filetransfer-storage_put_{job_id}",
        "files": [
          "*.webm"
        ]
      }
    }
  ]
}

storage_put_singlefile

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_put"
  ],
  "region": "aws:eu-west-1",
  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "location": "s3://{com-castlabs-vtk-itest-access-keys}@com.castlabs.vtk.itest/integration-test-storage/",
        "files": [
          "audio_eng_128k.webm"
        ]
      }
    },
    {
      "tool": "storage:put",
      "parameters": {
        "location": "s3://{com-castlabs-vtk-itest-access-keys}@com.castlabs.vtk.itest/OUT/filetransfer-storage_put_{job_id}",
        "files": [
          "audio_eng_128k.webm"
        ]
      }
    }
  ]
}

storage_put_toolrole

{
  "tags": [
    "itest",
    "filetransfer",
    "storage_put",
    "tool_role"
  ],
  "region": "aws:eu-west-1",

  "tasks": [
    {
      "tool": "storage:get",
      "parameters": {
        "role_arn": "arn:aws:iam::685872506356:role/video-toolkit-integration-test-staging",
        "location": "s3://com.castlabs.vtk.itest",
        "files": [
          "integration-test-storage/*"
        ]
      }
    },
    {
      "tool": "storage:put",
      "parameters": {
        "role_arn": "arn:aws:iam::685872506356:role/video-toolkit-integration-test-staging",
        "location": "s3://com.castlabs.vtk.itest/OUT/filetransfer-storage_put_{job_id}",
        "files": [
          "*.mp4"
        ]
      }
    }
  ]
}
Previous topic: Transcoding
Next topic: Packaging