AWS CLI

TBD

Useful Scripts

List all JSON files under a prefix

# List all JSON files under a prefix
for key in $(aws s3 ls s3://some_prefix --recursive | awk '{print $4}'); do
  echo -n "  $key: "
  aws s3api select-object-content \
    --bucket "some-bucket" \
    --key "$key" \
    --expression "SELECT COUNT(*) FROM S3Object" \
    --expression-type SQL \
    --input-serialization '{"JSON": {"TYPE": "LINES"}, "CompressionType": "NONE"}' \
    --output-serialization '{"JSON": {}}' \
    /dev/stdout | grep -o '"_1":[0-9]*' | awk -F: '{print $2}'
done