Distributed counters

Many realtime apps have documents that act as counters. For example, you might count 'likes' on a post, or 'favorites' of a specific item.

In Cloud Firestore, you can only update a single document about once per second, which might be too low for some high-traffic applications.

Solution: Distributed counters

To support more frequent counter updates, create a distributed counter. Each counter is a document with a subcollection of "shards," and the value of the counter is the sum of the value of the shards.

Write throughput increases linearly with the number of shards, so a distributed counter with 10 shards can handle 10x as many writes as a traditional counter.

// counters/${ID}
{
  "num_shards": NUM_SHARDS,
  "shards": [subcollection]
}

// counters/${ID}/shards/${NUM}
{
  "count": 123
}

// counters/${ID}
struct Counter {
    let numShards: Int

    init(numShards: Int) {
        self.numShards = numShards
    }
}

// counters/${ID}/shards/${NUM}
struct Shard {
    let count: Int

    init(count: Int) {
        self.count = count
    }
}
SolutionCountersViewController.swift

// counters/${ID}
@interface FIRCounter : NSObject
@property (nonatomic, readonly) NSInteger shardCount;
@end

@implementation FIRCounter
- (instancetype)initWithShardCount:(NSInteger)shardCount {
  self = [super init];
  if (self != nil) {
    _shardCount = shardCount;
  }
  return self;
}
@end

// counters/${ID}/shards/${NUM}
@interface FIRShard : NSObject
@property (nonatomic, readonly) NSInteger count;
@end

@implementation FIRShard
- (instancetype)initWithCount:(NSInteger)count {
  self = [super init];
  if (self != nil) {
    _count = count;
  }
  return self;
}
@end
FIRSolutionCountersViewController.m

// counters/${ID}
public class Counter {
    int numShards;

    public Counter(int numShards) {
        this.numShards = numShards;
    }
}

// counters/${ID}/shards/${NUM}
public class Shard {
    int count;

    public Shard(int count) {
        this.count = count;
    }
}
SolutionCounters.java

// counters/${ID}
data class Counter(var numShards: Int)

// counters/${ID}/shards/${NUM}
data class Shard(var count: Int)
SolutionCounters.kt

The following code initializes a distributed counter:

function createCounter(ref, num_shards) {
    var batch = db.batch();

    // Initialize the counter document
    batch.set(ref, { num_shards: num_shards });

    // Initialize each shard with count=0
    for (let i = 0; i < num_shards; i++) {
        let shardRef = ref.collection('shards').doc(i.toString());
        batch.set(shardRef, { count: 0 });
    }

    // Commit the write batch
    return batch.commit();
}
test.solution-counters.js

func createCounter(ref: DocumentReference, numShards: Int) {
    ref.setData(["numShards": numShards]){ (err) in
        for i in 0...numShards {
            ref.collection("shards").document(String(i)).setData(["count": 0])
        }
    }
}
SolutionCountersViewController.swift

- (void)createCounterAtReference:(FIRDocumentReference *)reference
                      shardCount:(NSInteger)shardCount {
  [reference setData:@{ @"numShards": @(shardCount) } completion:^(NSError * _Nullable error) {
    for (NSInteger i = 0; i < shardCount; i++) {
      NSString *shardName = [NSString stringWithFormat:@"%ld", (long)shardCount];
      [[[reference collectionWithPath:@"shards"] documentWithPath:shardName]
          setData:@{ @"count": @(0) }];
    }
  }];
}
FIRSolutionCountersViewController.m

public Task<Void> createCounter(final DocumentReference ref, final int numShards) {
    // Initialize the counter document, then initialize each shard.
    return ref.set(new Counter(numShards))
            .continueWithTask(new Continuation<Void, Task<Void>>() {
                @Override
                public Task<Void> then(@NonNull Task<Void> task) throws Exception {
                    if (!task.isSuccessful()) {
                        throw task.getException();
                    }

                    List<Task<Void>> tasks = new ArrayList<>();

                    // Initialize each shard with count=0
                    for (int i = 0; i < numShards; i++) {
                        Task<Void> makeShard = ref.collection("shards")
                                .document(String.valueOf(i))
                                .set(new Shard(0));

                        tasks.add(makeShard);
                    }

                    return Tasks.whenAll(tasks);
                }
            });
}
SolutionCounters.java

fun createCounter(ref: DocumentReference, numShards: Int): Task<Void> {
    // Initialize the counter document, then initialize each shard.
    return ref.set(Counter(numShards))
            .continueWithTask { task ->
                if (!task.isSuccessful) {
                    throw task.exception!!
                }

                val tasks = arrayListOf<Task<Void>>()

                // Initialize each shard with count=0
                for (i in 0 until numShards) {
                    val makeShard = ref.collection("shards")
                            .document(i.toString())
                            .set(Shard(0))

                    tasks.add(makeShard)
                }

                Tasks.whenAll(tasks)
            }
}
SolutionCounters.kt

To increment the counter, choose a random shard and increment the count in a transaction:

function incrementCounter(db, ref, num_shards) {
    // Select a shard of the counter at random
    const shard_id = Math.floor(Math.random() * num_shards).toString();
    const shard_ref = ref.collection('shards').doc(shard_id);

    // Update count in a transaction
    return db.runTransaction(t => {
        return t.get(shard_ref).then(doc => {
            const new_count = doc.data().count + 1;
            t.update(shard_ref, { count: new_count });
        });
    });
}
test.solution-counters.js

func incrementCounter(ref: DocumentReference, numShards: Int) {
    // Select a shard of the counter at random
    let shardId = Int(arc4random_uniform(UInt32(numShards)))
    let shardRef = ref.collection("shards").document(String(shardId))

    // Update count in a transaction
    db.runTransaction({ (transaction, errorPointer) -> Any? in
        do {
            let shardData = try transaction.getDocument(shardRef).data() ?? [:]
            let shardCount = shardData["count"] as! Int
            transaction.updateData(["count": shardCount + 1], forDocument: shardRef)
        } catch {
            // Error getting shard data
            // ...
        }

        return nil
    }) { (object, err) in
        // ...
    }
}
SolutionCountersViewController.swift

- (void)incrementCounterAtReference:(FIRDocumentReference *)reference
                         shardCount:(NSInteger)shardCount {
  // Select a shard of the counter at random
  NSInteger shardID = (NSInteger)arc4random_uniform((uint32_t)shardCount);
  NSString *shardName = [NSString stringWithFormat:@"%ld", (long)shardID];
  FIRDocumentReference *shardReference =
      [[reference collectionWithPath:@"shards"] documentWithPath:shardName];

  // Update count in a transaction
  [self.db runTransactionWithBlock:^id (FIRTransaction *transaction, NSError **errorPointer) {
    FIRDocumentSnapshot *shardSnapshot =
        [transaction getDocument:shardReference error:errorPointer];
    if (shardSnapshot == nil) {
      return nil;
    }
    NSInteger shardValue = [shardSnapshot[@"count"] integerValue];
    [transaction updateData:@{ @"count": @(shardValue + 1) } forDocument:shardReference];
    return nil;
  } completion:^(id result, NSError *error) {
    // ...
  }];
}
FIRSolutionCountersViewController.m

public Task<Void> incrementCounter(final DocumentReference ref, final int numShards) {
    int shardId = (int) Math.floor(Math.random() * numShards);
    final DocumentReference shardRef = ref.collection("shards").document(String.valueOf(shardId));

    return db.runTransaction(new Transaction.Function<Void>() {
        @Override
        public Void apply(Transaction transaction) throws FirebaseFirestoreException {
            Shard shard = transaction.get(shardRef).toObject(Shard.class);
            shard.count += 1;

            transaction.set(shardRef, shard);
            return null;
        }
    });
}
SolutionCounters.java

fun incrementCounter(ref: DocumentReference, numShards: Int): Task<Void> {
    val shardId = Math.floor(Math.random() * numShards).toInt()
    val shardRef = ref.collection("shards").document(shardId.toString())

    return db.runTransaction { transaction ->
        val shard = transaction.get(shardRef).toObject(Shard::class.java)
        shard!!.count += 1

        transaction.set(shardRef, shard!!)
        null
    }
}
SolutionCounters.kt

To get the total count, query for all shards and sum their count fields:

function getCount(ref) {
    // Sum the count of each shard in the subcollection
    return ref.collection('shards').get().then(snapshot => {
        let total_count = 0;
        snapshot.forEach(doc => {
            total_count += doc.data().count;
        });

        return total_count;
    });
}
test.solution-counters.js

func getCount(ref: DocumentReference) {
    ref.collection("shards").getDocuments() { (querySnapshot, err) in
        var totalCount = 0
        if err != nil {
            // Error getting shards
            // ...
        } else {
            for document in querySnapshot!.documents {
                let count = document.data()["count"] as! Int
                totalCount += count
            }
        }

        print("Total count is \(totalCount)")
    }
}
SolutionCountersViewController.swift

- (void)getCountWithReference:(FIRDocumentReference *)reference {
  [[reference collectionWithPath:@"shards"]
      getDocumentsWithCompletion:^(FIRQuerySnapshot *snapshot,
                                   NSError *error) {
        NSInteger totalCount = 0;
        if (error != nil) {
          // Error getting shards
          // ...
        } else {
          for (FIRDocumentSnapshot *document in snapshot.documents) {
            NSInteger count = [document[@"count"] integerValue];
            totalCount += count;
          }

          NSLog(@"Total count is %ld", (long)totalCount);
        }
  }];
}
FIRSolutionCountersViewController.m

public Task<Integer> getCount(final DocumentReference ref) {
    // Sum the count of each shard in the subcollection
    return ref.collection("shards").get()
            .continueWith(new Continuation<QuerySnapshot, Integer>() {
                @Override
                public Integer then(@NonNull Task<QuerySnapshot> task) throws Exception {
                    int count = 0;
                    for (DocumentSnapshot snap : task.getResult()) {
                        Shard shard = snap.toObject(Shard.class);
                        count += shard.count;
                    }
                    return count;
                }
            });
}
SolutionCounters.java

fun getCount(ref: DocumentReference): Task<Int> {
    // Sum the count of each shard in the subcollection
    return ref.collection("shards").get()
            .continueWith { task ->
                var count = 0
                for (snap in task.result!!) {
                    val shard = snap.toObject(Shard::class.java)
                    count += shard.count
                }
                count
            }
}
SolutionCounters.kt

Limitations

The solution shown above is a scalable way to create shared counters in Cloud Firestore, but you should be aware of the following limitations:

• Shard count - The number of shards controls the performance of the distributed counter. With too few shards, some transactions may have to retry before succeeding, which will slow writes. With too many shards, reads become slower and more expensive.
• Cost - The cost of reading a counter value increases linearly with the number of shards, because the entire shards subcollection must be loaded.