Server.Import.WatchS3Directory

This is an artifact that will monitor an S3 path for collections, which it will then ingest.

name: Server.Import.WatchS3Directory
description: |
   This is an artifact that will monitor an S3 path for collections, 
   which it will then ingest. 

# Can be CLIENT, CLIENT_EVENT, SERVER, SERVER_EVENT
type: SERVER_EVENT

parameters:
   - name: WatchDir
     default: "/velociraptor/*.zip"

   - name: Endpoint
     default: 'http://127.0.0.1:9000/'
     
   - name: Key
     default: 'admin'
     
   - name: Secret
     default: 'password'
     
   - name: Region
     default: 'us-east-1'

sources:
  - query: |
        LET S3_CREDENTIALS <= dict(
            endpoint=Endpoint, 
            credentials_key=Key, 
            credentials_secret=Secret,
            region=Region,
            no_verify_cert=1)
        SELECT * FROM foreach(
            row={
                SELECT * FROM diff(
                    query={
                        SELECT OSPath FROM glob(globs=WatchDir, accessor="s3")
                    }, period=3, key="OSPath"
                )
                WHERE Diff =~ "added"
            }, query={
                SELECT *, import_collection(
                    filename=OSPath,
                    accessor="s3"
                ),  OSPath 
                FROM scope()
            }
        )