Tech Insights: Personal Tech Blog

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

// Example: Schema flexibility in action
const orderItem = {
  PK: 'USER#12345',
  SK: 'ORDER#67890',
  type: 'order',
  amount: 99.99,
  currency: 'USD',
  items: [
    { productId: 'P001', quantity: 2 }
  ],
  createdAt: '2025-10-09T10:00:00Z'
};

// Later, add new attributes without schema changes
const orderWithShipping = {
  ...orderItem,
  shippingAddress: {
    street: '123 Main St',
    city: 'Seattle',
    state: 'WA',
    zip: '98101'
  },
  trackingNumber: 'TRACK123'
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

// Generate partition key with random suffix to distribute writes
const generateShardedKey = (baseKey, shardCount = 10) => {
  const shard = Math.floor(Math.random() * shardCount);
  return `${baseKey}#SHARD${shard}`;
};

const createDistributedItem = async (userId, data) => {
  const item = {
    PK: generateShardedKey(`USER#${userId}`),
    SK: `DATA#${Date.now()}`,
    userId, // Preserve for queries
    ...data
  };
  
  await docClient.send(new PutCommand({
    TableName: 'MyTable',
    Item: item
  }));
};

// Query across all shards
const queryAllShards = async (userId, shardCount = 10) => {
  const promises = [];
  
  for (let i = 0; i < shardCount; i++) {
    const pk = `USER#${userId}#SHARD${i}`;
    promises.push(
      docClient.send(new QueryCommand({
        TableName: 'MyTable',
        KeyConditionExpression: 'PK = :pk',
        ExpressionAttributeValues: { ':pk': pk }
      }))
    );
  }
  
  const results = await Promise.all(promises);
  return results.flatMap(r => r.Items || []);
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb';

// Production configuration with observability and resilience
const client = new DynamoDBClient({
  region: process.env.AWS_REGION || 'us-east-1',
  maxAttempts: 3,
  retryMode: 'adaptive', // AWS SDK v3 adaptive retry with exponential backoff
  requestHandler: {
    connectionTimeout: 3000,
    socketTimeout: 5000
  },
  logger: console // Enable SDK debug logging
});

const docClient = DynamoDBDocumentClient.from(client, {
  marshallOptions: {
    removeUndefinedValues: true, // Prevent ValidationException
    convertEmptyValues: true, // Convert empty strings to null
    convertClassInstanceToMap: true // Serialize class instances
  },
  unmarshallOptions: {
    wrapNumbers: false // Return numbers as JavaScript numbers (precision loss possible)
  }
});

export { client, docClient };

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

import { CreateTableCommand } from '@aws-sdk/client-dynamodb';

const createProductionTable = async () => {
  const params = {
    TableName: 'Movies',
    KeySchema: [
      { AttributeName: 'movieId', KeyType: 'HASH' }
    ],
    AttributeDefinitions: [
      { AttributeName: 'movieId', AttributeType: 'S' },
      { AttributeName: 'year', AttributeType: 'N' },
      { AttributeName: 'genre', AttributeType: 'S' },
      { AttributeName: 'rating', AttributeType: 'N' }
    ],
    GlobalSecondaryIndexes: [
      {
        IndexName: 'YearIndex',
        KeySchema: [
          { AttributeName: 'year', KeyType: 'HASH' }
        ],
        Projection: {
          ProjectionType: 'ALL' // Include all attributes (higher storage cost)
        },
        ProvisionedThroughput: {
          ReadCapacityUnits: 5,
          WriteCapacityUnits: 5
        }
      },
      {
        IndexName: 'GenreYearIndex',
        KeySchema: [
          { AttributeName: 'genre', KeyType: 'HASH' },
          { AttributeName: 'year', KeyType: 'RANGE' }
        ],
        Projection: {
          ProjectionType: 'KEYS_ONLY' // Only include key attributes (lower cost)
        },
        ProvisionedThroughput: {
          ReadCapacityUnits: 5,
          WriteCapacityUnits: 5
        }
        },
      {
        IndexName: 'RatingIndex',
        KeySchema: [
          { AttributeName: 'rating', KeyType: 'HASH' }
    ],
        Projection: {
          ProjectionType: 'INCLUDE',
          NonKeyAttributes: ['title', 'director', 'year'] // Selective attributes
        },
    ProvisionedThroughput: {
          ReadCapacityUnits: 5,
          WriteCapacityUnits: 5
        }
      }
    ],
    BillingMode: 'PROVISIONED', // Use 'PAY_PER_REQUEST' for on-demand
    StreamSpecification: {
      StreamEnabled: true,
      StreamViewType: 'NEW_AND_OLD_IMAGES' // Capture before/after for CDC
    },
    SSESpecification: {
      Enabled: true,
      SSEType: 'KMS', // Use AWS KMS for encryption
      KMSMasterKeyId: process.env.KMS_KEY_ID
    },
    PointInTimeRecoverySpecification: {
      PointInTimeRecoveryEnabled: true
    },
    Tags: [
      { Key: 'Environment', Value: 'Production' },
      { Key: 'CostCenter', Value: 'Engineering' },
      { Key: 'Application', Value: 'MovieCatalog' }
    ]
  };

  try {
    const command = new CreateTableCommand(params);
    const result = await client.send(command);
    
    console.log('Table created:', {
      tableName: result.TableDescription.TableName,
      status: result.TableDescription.TableStatus,
      arn: result.TableDescription.TableArn,
      itemCount: result.TableDescription.ItemCount
    });
    
    return result;
  } catch (error) {
    if (error.name === 'ResourceInUseException') {
      console.log('Table already exists');
    } else {
      console.error('Table creation failed:', error);
    throw error;
    }
  }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

import { PutCommand } from '@aws-sdk/lib-dynamodb';
import { randomUUID } from 'crypto';

const createMovieWithIdempotency = async (movieData) => {
  const item = {
    movieId: movieData.movieId || randomUUID(),
      title: movieData.title,
      year: movieData.year,
      genre: movieData.genre,
      director: movieData.director,
    rating: movieData.rating,
      createdAt: new Date().toISOString(),
      updatedAt: new Date().toISOString(),
    version: 1 // Optimistic locking
  };

  const command = new PutCommand({
    TableName: 'Movies',
    Item: item,
    ConditionExpression: 'attribute_not_exists(movieId)', // Prevent overwrites
    ReturnValues: 'NONE' // Reduce response payload size
  });

  try {
    await docClient.send(command);
    console.log('Movie created:', item.movieId);
    return item;
  } catch (error) {
    if (error.name === 'ConditionalCheckFailedException') {
      console.warn('Movie already exists:', item.movieId);
      // Return existing item or throw custom error
      throw new Error(`Movie with ID ${item.movieId} already exists`);
    }
    throw error;
  }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

import { QueryCommand } from '@aws-sdk/lib-dynamodb';

const queryMoviesByGenrePaginated = async (genre, options = {}) => {
  const {
    limit = 50,
    lastEvaluatedKey = null,
    consistentRead = false,
    yearFrom = null,
    yearTo = null
  } = options;

  let keyConditionExpression = 'genre = :genre';
  const expressionAttributeValues = { ':genre': genre };

  // Add range key condition if year filters provided
  if (yearFrom && yearTo) {
    keyConditionExpression += ' AND #year BETWEEN :yearFrom AND :yearTo';
    expressionAttributeValues[':yearFrom'] = yearFrom;
    expressionAttributeValues[':yearTo'] = yearTo;
  }

  const params = {
    TableName: 'Movies',
    IndexName: 'GenreYearIndex',
    KeyConditionExpression: keyConditionExpression,
    ExpressionAttributeNames: {
      '#year': 'year' // Handle reserved keywords
    },
    ExpressionAttributeValues: expressionAttributeValues,
    Limit: limit,
    ConsistentRead: consistentRead, // GSI queries are eventually consistent only
    ScanIndexForward: false // Sort descending by year
  };

  if (lastEvaluatedKey) {
    params.ExclusiveStartKey = lastEvaluatedKey;
  }

  try {
    const command = new QueryCommand(params);
    const response = await docClient.send(command);
    
    return {
      items: response.Items || [],
      count: response.Count,
      scannedCount: response.ScannedCount,
      lastEvaluatedKey: response.LastEvaluatedKey,
      hasMore: !!response.LastEvaluatedKey,
      consumedCapacity: response.ConsumedCapacity
    };
  } catch (error) {
    console.error('Query failed:', error);
    throw error;
  }
};

// Usage: Query all pages
const getAllMoviesByGenre = async (genre) => {
  const allItems = [];
  let lastEvaluatedKey = null;
  
  do {
    const result = await queryMoviesByGenrePaginated(genre, {
      limit: 100,
      lastEvaluatedKey
    });
    
    allItems.push(...result.items);
    lastEvaluatedKey = result.lastEvaluatedKey;
  } while (lastEvaluatedKey);
  
  return allItems;
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

import { UpdateCommand } from '@aws-sdk/lib-dynamodb';

const updateMovieWithOptimisticLock = async (movieId, updates, expectedVersion) => {
  const command = new UpdateCommand({
    TableName: 'Movies',
    Key: { movieId },
    UpdateExpression: `
      SET #title = :title,
          #rating = :rating,
          #updatedAt = :updatedAt,
          #version = :newVersion
      ADD #viewCount :increment
    `,
    ExpressionAttributeNames: {
      '#title': 'title',
      '#rating': 'rating',
      '#updatedAt': 'updatedAt',
      '#version': 'version',
      '#viewCount': 'viewCount'
    },
    ExpressionAttributeValues: {
      ':title': updates.title,
      ':rating': updates.rating,
      ':updatedAt': new Date().toISOString(),
      ':newVersion': expectedVersion + 1,
      ':increment': 1,
      ':expectedVersion': expectedVersion
    },
    ConditionExpression: 'attribute_exists(movieId) AND #version = :expectedVersion',
    ReturnValues: 'ALL_NEW',
    ReturnConsumedCapacity: 'TOTAL'
  });

  try {
    const response = await docClient.send(command);
    console.log('Movie updated:', response.Attributes);
    console.log('Consumed capacity:', response.ConsumedCapacity);
    return response.Attributes;
  } catch (error) {
    if (error.name === 'ConditionalCheckFailedException') {
      throw new Error('Version conflict: Movie was modified by another process');
    }
    throw error;
  }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

import { TransactWriteCommand } from '@aws-sdk/lib-dynamodb';

const processOrderWithInventory = async (orderId, userId, items) => {
  const transactItems = [];

  // Create order record
  transactItems.push({
    Put: {
      TableName: 'Orders',
      Item: {
        orderId,
        userId,
        items,
        status: 'pending',
        createdAt: new Date().toISOString(),
        total: items.reduce((sum, item) => sum + item.price * item.quantity, 0)
      },
      ConditionExpression: 'attribute_not_exists(orderId)'
    }
  });

  // Decrement inventory for each item
  items.forEach(item => {
    transactItems.push({
      Update: {
        TableName: 'Inventory',
        Key: { productId: item.productId },
        UpdateExpression: 'SET #stock = #stock - :quantity',
    ExpressionAttributeNames: {
          '#stock': 'stock'
    },
    ExpressionAttributeValues: {
          ':quantity': item.quantity,
          ':minStock': 0
        },
        ConditionExpression: '#stock >= :quantity' // Prevent overselling
      }
    });
  });

  const command = new TransactWriteCommand({
    TransactItems: transactItems,
    ReturnConsumedCapacity: 'TOTAL',
    ReturnItemCollectionMetrics: 'SIZE'
  });

  try {
    const response = await docClient.send(command);
    console.log('Transaction successful:', {
      consumedCapacity: response.ConsumedCapacity,
      itemCollectionMetrics: response.ItemCollectionMetrics
    });
    return { orderId, status: 'success' };
  } catch (error) {
    if (error.name === 'TransactionCanceledException') {
      console.error('Transaction failed:', error.CancellationReasons);
      throw new Error('Order processing failed: Insufficient inventory or order exists');
    }
    throw error;
  }
};

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101

import { BatchWriteCommand, BatchGetCommand } from '@aws-sdk/lib-dynamodb';

const batchWriteMovies = async (movies) => {
  const BATCH_SIZE = 25; // DynamoDB limit
  const batches = [];
  
  // Split into batches of 25
  for (let i = 0; i < movies.length; i += BATCH_SIZE) {
    batches.push(movies.slice(i, i + BATCH_SIZE));
  }

  const results = [];
  const unprocessedItems = [];

  for (const batch of batches) {
    const command = new BatchWriteCommand({
      RequestItems: {
        Movies: batch.map(movie => ({
          PutRequest: { Item: movie }
        }))
      },
      ReturnConsumedCapacity: 'TOTAL'
  });

  try {
    const response = await docClient.send(command);
      results.push(response);
      
      // Handle unprocessed items with exponential backoff
      if (response.UnprocessedItems?.Movies) {
        unprocessedItems.push(...response.UnprocessedItems.Movies);
      }
  } catch (error) {
      console.error('Batch write error:', error);
      throw error;
    }
  }

  // Retry unprocessed items
  if (unprocessedItems.length > 0) {
    console.log(`Retrying ${unprocessedItems.length} unprocessed items`);
    await new Promise(resolve => setTimeout(resolve, 1000)); // 1 second delay
    
    const retryCommand = new BatchWriteCommand({
      RequestItems: { Movies: unprocessedItems }
    });
    
    await docClient.send(retryCommand);
  }

  return {
    processed: movies.length - unprocessedItems.length,
    unprocessed: unprocessedItems.length,
    totalConsumedCapacity: results.reduce((sum, r) => 
      sum + (r.ConsumedCapacity?.[0]?.CapacityUnits || 0), 0
    )
  };
};

const batchGetMovies = async (movieIds) => {
  const BATCH_SIZE = 100; // DynamoDB limit
  const batches = [];
  
  for (let i = 0; i < movieIds.length; i += BATCH_SIZE) {
    batches.push(movieIds.slice(i, i + BATCH_SIZE));
  }

  const allItems = [];

  for (const batch of batches) {
    const command = new BatchGetCommand({
      RequestItems: {
        Movies: {
          Keys: batch.map(id => ({ movieId: id })),
          ProjectionExpression: 'movieId, title, #year, genre, rating',
          ExpressionAttributeNames: {
            '#year': 'year'
          }
        }
      },
      ReturnConsumedCapacity: 'TOTAL'
    });

    try {
      const response = await docClient.send(command);
      allItems.push(...(response.Responses?.Movies || []));
      
      // Handle unprocessed keys
      if (response.UnprocessedKeys?.Movies) {
        console.log('Unprocessed keys detected, retrying...');
        await new Promise(resolve => setTimeout(resolve, 500));
        // Implement retry logic here
      }
    } catch (error) {
      console.error('Batch get error:', error);
    throw error;
  }
  }

  return allItems;
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

// Entity definitions with composite keys
const createUser = (userId, userData) => ({
  PK: `USER#${userId}`,
  SK: `METADATA`,
  type: 'user',
  userId,
  email: userData.email,
  name: userData.name,
  createdAt: new Date().toISOString(),
  GSI1PK: `EMAIL#${userData.email}`,
  GSI1SK: `USER#${userId}`
});

const createOrder = (userId, orderId, orderData) => ({
  PK: `USER#${userId}`,
  SK: `ORDER#${orderId}`,
  type: 'order',
  orderId,
  userId,
  amount: orderData.amount,
  status: orderData.status,
  createdAt: new Date().toISOString(),
  GSI1PK: `ORDER#${orderId}`,
  GSI1SK: `USER#${userId}`
});

const createProduct = (productId, productData) => ({
  PK: `PRODUCT#${productId}`,
  SK: `METADATA`,
  type: 'product',
  productId,
  name: productData.name,
  price: productData.price,
  category: productData.category,
  GSI1PK: `CATEGORY#${productData.category}`,
  GSI1SK: `PRODUCT#${productId}`
});

// Query patterns enabled by single-table design
const getUserWithOrders = async (userId) => {
  const command = new QueryCommand({
    TableName: 'AppData',
    KeyConditionExpression: 'PK = :pk',
    ExpressionAttributeValues: {
      ':pk': `USER#${userId}`
    }
  });

  const response = await docClient.send(command);
  
  const user = response.Items.find(item => item.type === 'user');
  const orders = response.Items.filter(item => item.type === 'order');
  
  return { user, orders };
};

const getOrderWithUser = async (orderId) => {
  const command = new QueryCommand({
    TableName: 'AppData',
    IndexName: 'GSI1',
    KeyConditionExpression: 'GSI1PK = :pk',
    ExpressionAttributeValues: {
      ':pk': `ORDER#${orderId}`
    }
  });

  const response = await docClient.send(command);
  return response.Items;
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

const getMovieOptimized = async (movieId) => {
  const command = new GetCommand({
    TableName: 'Movies',
    Key: { movieId },
    ProjectionExpression: 'movieId, title, #year, rating',
    ExpressionAttributeNames: {
      '#year': 'year'
    }
  });

  const response = await docClient.send(command);
  return response.Item;
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

// Eventually consistent read (default, 50% cheaper)
const getItemEventuallyConsistent = async (key) => {
  return docClient.send(new GetCommand({
    TableName: 'Movies',
    Key: key,
    ConsistentRead: false // Default
  }));
};

// Strongly consistent read (2× cost, guaranteed latest)
const getItemStronglyConsistent = async (key) => {
  return docClient.send(new GetCommand({
    TableName: 'Movies',
    Key: key,
    ConsistentRead: true
  }));
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

const updateMultipleMovies = async (updates) => {
  const promises = updates.map(update =>
    docClient.send(new UpdateCommand({
      TableName: 'Movies',
      Key: { movieId: update.movieId },
      UpdateExpression: 'SET #rating = :rating',
      ExpressionAttributeNames: { '#rating': 'rating' },
      ExpressionAttributeValues: { ':rating': update.rating }
    }))
  );

  try {
    const results = await Promise.allSettled(promises);
    
    const successful = results.filter(r => r.status === 'fulfilled').length;
    const failed = results.filter(r => r.status === 'rejected').length;
    
    console.log(`Updated ${successful} movies, ${failed} failures`);
    return { successful, failed };
    } catch (error) {
    console.error('Parallel write error:', error);
      throw error;
    }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

const writeWithSharding = async (baseKey, data, shardCount = 10) => {
  const shard = Math.floor(Math.random() * shardCount);
  const shardedKey = `${baseKey}#${shard}`;
  
  await docClient.send(new PutCommand({
    TableName: 'HighVolumeTable',
    Item: {
      PK: shardedKey,
      SK: `DATA#${Date.now()}`,
      baseKey, // Preserve for aggregation queries
      ...data
    }
  }));
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

import { createClient } from 'redis';

const redis = createClient({
  url: process.env.REDIS_URL
});

await redis.connect();

const getCachedMovie = async (movieId) => {
  const cacheKey = `movie:${movieId}`;
  
  // Check cache first
  const cached = await redis.get(cacheKey);
  if (cached) {
    console.log('Cache hit');
    return JSON.parse(cached);
  }
  
  // Cache miss: fetch from DynamoDB
  console.log('Cache miss');
  const movie = await getMovieOptimized(movieId);
  
  if (movie) {
    // Cache for 1 hour
    await redis.setEx(cacheKey, 3600, JSON.stringify(movie));
  }
  
  return movie;
};

// Cache invalidation on update
const updateMovieWithCacheInvalidation = async (movieId, updates) => {
  const updated = await updateMovieWithOptimisticLock(movieId, updates, 1);
  
  // Invalidate cache
  await redis.del(`movie:${movieId}`);
  
  return updated;
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

import { CloudWatchClient, PutMetricDataCommand } from '@aws-sdk/client-cloudwatch';

const cloudwatch = new CloudWatchClient({ region: 'us-east-1' });

const publishMetric = async (metricName, value, unit = 'Count') => {
  const command = new PutMetricDataCommand({
    Namespace: 'MyApp/DynamoDB',
    MetricData: [
      {
        MetricName: metricName,
        Value: value,
        Unit: unit,
        Timestamp: new Date(),
        Dimensions: [
          { Name: 'Environment', Value: process.env.NODE_ENV },
          { Name: 'TableName', Value: 'Movies' }
        ]
      }
    ]
  });

  try {
    await cloudwatch.send(command);
  } catch (error) {
    console.error('Failed to publish metric:', error);
  }
};

// Instrumented operation
const getMovieWithMetrics = async (movieId) => {
  const startTime = Date.now();
  
  try {
    const movie = await getMovieOptimized(movieId);
    const duration = Date.now() - startTime;
    
    await Promise.all([
      publishMetric('GetMovie.Success', 1),
      publishMetric('GetMovie.Duration', duration, 'Milliseconds'),
      publishMetric('GetMovie.CacheHit', movie ? 1 : 0)
    ]);
    
    return movie;
  } catch (error) {
    await publishMetric('GetMovie.Error', 1);
    throw error;
  }
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

const handleDynamoDBError = async (operation, maxRetries = 3) => {
  let retryCount = 0;
  
  while (retryCount < maxRetries) {
    try {
      return await operation();
    } catch (error) {
      console.error(`Attempt ${retryCount + 1} failed:`, error.name);
      
      switch (error.name) {
        case 'ProvisionedThroughputExceededException':
        case 'ThrottlingException':
          // Exponential backoff
          const delay = Math.min(1000 * Math.pow(2, retryCount), 10000);
          console.log(`Throttled, retrying in ${delay}ms`);
          await new Promise(resolve => setTimeout(resolve, delay));
          retryCount++;
          break;
          
        case 'ConditionalCheckFailedException':
          // Business logic error, don't retry
          throw new Error('Conditional check failed');
          
        case 'ValidationException':
          // Client error, don't retry
          throw new Error(`Validation failed: ${error.message}`);
          
        case 'ResourceNotFoundException':
          throw new Error('Table not found');
          
        case 'ItemCollectionSizeLimitExceededException':
          throw new Error('Item collection too large (>10GB)');
          
        case 'TransactionConflictException':
          // Concurrent transaction conflict, retry
          await new Promise(resolve => setTimeout(resolve, 100));
          retryCount++;
          break;
          
        case 'InternalServerError':
        case 'ServiceUnavailable':
          // AWS service issue, retry with backoff
          await new Promise(resolve => setTimeout(resolve, 1000));
          retryCount++;
          break;
          
        default:
          throw error;
      }
    }
  }
  
  throw new Error(`Operation failed after ${maxRetries} retries`);
};

// Usage
const resilientGetMovie = (movieId) => 
  handleDynamoDBError(() => getMovieOptimized(movieId));

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

class CircuitBreaker {
  constructor(threshold = 5, timeout = 60000) {
    this.failureCount = 0;
    this.threshold = threshold;
    this.timeout = timeout;
    this.state = 'CLOSED'; // CLOSED, OPEN, HALF_OPEN
    this.nextAttempt = Date.now();
  }

  async execute(operation) {
    if (this.state === 'OPEN') {
      if (Date.now() < this.nextAttempt) {
        throw new Error('Circuit breaker is OPEN');
      }
      this.state = 'HALF_OPEN';
    }

    try {
      const result = await operation();
      this.onSuccess();
    return result;
  } catch (error) {
      this.onFailure();
    throw error;
  }
  }

  onSuccess() {
    this.failureCount = 0;
    this.state = 'CLOSED';
  }

  onFailure() {
    this.failureCount++;
    if (this.failureCount >= this.threshold) {
      this.state = 'OPEN';
      this.nextAttempt = Date.now() + this.timeout;
      console.log(`Circuit breaker opened for ${this.timeout}ms`);
    }
  }
}

const breaker = new CircuitBreaker(5, 60000);

const protectedGetMovie = async (movieId) => {
  return breaker.execute(() => getMovieOptimized(movieId));
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

Is traffic predictable?
├── Yes
│   └── Provisioned with Auto-Scaling
│       └── Save 40-60% vs On-Demand
└── No
    ├── Is traffic intermittent (<10% of time)?
    │   └── On-Demand
    │       └── Pay only for actual usage
    └── Is traffic sustained but variable?
        └── Provisioned with aggressive auto-scaling
            └── Balance cost and performance

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';

const s3 = new S3Client({ region: 'us-east-1' });

const archiveOldMovies = async (cutoffYear) => {
  let lastEvaluatedKey = null;
  let archivedCount = 0;

  do {
    // Query movies older than cutoff year
    const result = await docClient.send(new QueryCommand({
      TableName: 'Movies',
      IndexName: 'YearIndex',
      KeyConditionExpression: '#year < :cutoffYear',
      ExpressionAttributeNames: { '#year': 'year' },
      ExpressionAttributeValues: { ':cutoffYear': cutoffYear },
      ExclusiveStartKey: lastEvaluatedKey
    }));

    if (result.Items.length > 0) {
      // Archive to S3
      const archiveKey = `archive/movies-${Date.now()}.json`;
      await s3.send(new PutObjectCommand({
        Bucket: 'my-archive-bucket',
        Key: archiveKey,
        Body: JSON.stringify(result.Items),
        StorageClass: 'GLACIER_IR' // Instant retrieval glacier
      }));
  
  // Delete from DynamoDB
      for (const movie of result.Items) {
        await docClient.send(new DeleteCommand({
          TableName: 'Movies',
          Key: { movieId: movie.movieId }
        }));
      }

      archivedCount += result.Items.length;
    }

    lastEvaluatedKey = result.LastEvaluatedKey;
  } while (lastEvaluatedKey);

  console.log(`Archived ${archivedCount} movies to S3`);
  return archivedCount;
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

// 1. Use sparse indexes to reduce index size
const createSparseIndex = {
  IndexName: 'ActiveUsersIndex',
  KeySchema: [{ AttributeName: 'status', KeyType: 'HASH' }],
  Projection: { ProjectionType: 'KEYS_ONLY' },
  // Only items with 'status' attribute are indexed
};

// 2. Use KEYS_ONLY projection when full attributes not needed
// Reduces GSI storage by 70-90%

// 3. Compress large attributes before storage
import { gzip, gunzip } from 'zlib';
import { promisify } from 'util';

const gzipAsync = promisify(gzip);
const gunzipAsync = promisify(gunzip);

const putCompressedItem = async (item) => {
  const largeData = JSON.stringify(item.metadata);
  const compressed = await gzipAsync(largeData);
  
  await docClient.send(new PutCommand({
    TableName: 'Movies',
    Item: {
      ...item,
      metadata: compressed.toString('base64'),
      compressed: true
    }
  }));
};

// 4. Use TTL to automatically delete expired items
// Add ttl attribute with Unix timestamp
const itemWithTTL = {
  movieId: 'movie123',
  title: 'Temporary Movie',
  ttl: Math.floor(Date.now() / 1000) + (86400 * 30) // 30 days
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "dynamodb:GetItem",
        "dynamodb:Query"
      ],
      "Resource": "arn:aws:dynamodb:us-east-1:123456789012:table/Movies",
      "Condition": {
        "ForAllValues:StringEquals": {
          "dynamodb:LeadingKeys": ["${aws:userid}"]
        }
      }
    },
    {
      "Effect": "Allow",
      "Action": [
        "dynamodb:GetItem",
        "dynamodb:Query"
      ],
      "Resource": "arn:aws:dynamodb:us-east-1:123456789012:table/Movies/index/*"
    }
  ]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "dynamodb:GetItem",
        "dynamodb:PutItem",
        "dynamodb:UpdateItem",
        "dynamodb:DeleteItem"
      ],
      "Resource": "arn:aws:dynamodb:us-east-1:123456789012:table/Movies",
      "Condition": {
        "ForAllValues:StringEquals": {
          "dynamodb:Attributes": [
            "movieId",
            "title",
            "year",
            "genre"
          ]
        },
        "StringEqualsIfExists": {
          "dynamodb:Select": "SPECIFIC_ATTRIBUTES"
        }
      }
    }
  ]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

// Client configuration for VPC endpoint
const vpcClient = new DynamoDBClient({
  region: 'us-east-1',
  endpoint: 'https://vpce-1234567-abcdefgh.dynamodb.us-east-1.vpce.amazonaws.com',
  tls: true
});

// No internet gateway required
// Reduces data transfer costs
// Improves security posture

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

import { KMSClient, GenerateDataKeyCommand } from '@aws-sdk/client-kms';

const kms = new KMSClient({ region: 'us-east-1' });

// Client-side encryption for sensitive data
const encryptSensitiveData = async (data, keyId) => {
  const command = new GenerateDataKeyCommand({
    KeyId: keyId,
    KeySpec: 'AES_256'
  });

  const { Plaintext, CiphertextBlob } = await kms.send(command);
  
  // Use Plaintext to encrypt data (implementation depends on crypto library)
  const encrypted = encrypt(data, Plaintext);
  
  return {
    encryptedData: encrypted,
    encryptedKey: CiphertextBlob.toString('base64')
  };
};

// Store encrypted data in DynamoDB
const putEncryptedMovie = async (movie) => {
  const encrypted = await encryptSensitiveData(
    movie.sensitiveField,
    process.env.KMS_KEY_ID
  );
  
  await docClient.send(new PutCommand({
    TableName: 'Movies',
    Item: {
      ...movie,
      sensitiveField: encrypted.encryptedData,
      encryptionKey: encrypted.encryptedKey
    }
  }));
};