ts-datastore-orm (Typescript Orm wrapper for Google Datastore)

ts-datastore-orm targets to provide a strong typed and structural ORM feature for Datastore (Firestore in Datastore mode).

Please check the examples below to check out all the amazing features!

This package has 0 dependencies. You have to install the @google-cloud/datastore manually.

npm install -s @google-cloud/datastore@latest

This package is compatible with @google-cloud/datastore v5.0.0 or above. I have tested the library specifically with the following versions: v5.0.6, v5.1.0, v6.1.1, v6.2.0, v6.3.0 v7.0.0

You can also compare the native performance of @google-cloud/datastore with ts-datastorem-orm. Basically there is no significant overhead compared with native nodejs-datastore package.

Project Setup

• npm install -s @google-cloud/datastore@latest
• npm install -s ts-datastore-orm@latest
• In tsconfig.json
  • set "experimentalDecorators" to true.
  • set "emitDecoratorMetadata" to true.
  • set "strictNullChecks" to true.
• Generate service-account.json from Goolge APIs. (Details won't be covered here)

Example: connection

async function connectionExamples() {
  const connection1 = await createConnection({keyFilename: "./datastoreServiceAccount.json"});
  const connection2 = await createConnection({clientEmail: "", privateKey: ""});
  const datastore1 = connection1.datastore;
  const datastore2 = connection2.datastore;
}

Example: define Entity

import { BaseEntity, CompositeIndex, CompositeIndexExporter, createConnection, Entity, Field, 
tsDatastoreOrm, TsDatastoreOrmError, BeforeDelete, BeforeInsert, BeforeUpdate, BeforeUpsert, AfterLoad} from "ts-datastore-orm";

@CompositeIndex({_id: "desc"})
@Entity({namespace: "testing", kind: "User", enumerable: true})
export class User extends BaseEntity {
    @Field({generateId: true})
    public _id: number = 0;

    @Field()
    public date: Date = new Date();

    @Field({index: true})
    public string: string = "";

    @Field()
    public number: number = 10;

    @Field()
    public buffer: Buffer = Buffer.alloc(1);

    @Field()
    public array: number[] = [];

    @Field({index: true, excludeFromIndexes: ["object.name"]})
    public object: any = {};

    @Field()
    public undefined: undefined = undefined;

    @Field()
    public null: null = null;
}

@CompositeIndex({number: "desc", name: "desc"})
@CompositeIndex({_id: "desc"})
@Entity() // namespace: default, kind: same as class name
export class TaskGroup extends BaseEntity {
    @Field({generateId: true})
    public _id: number = 0;

    @Field()
    public name: string = "";

    @Field()
    public number: number = 0;

    @AfterLoad()
    @BeforeInsert()
    @BeforeUpsert()
    @BeforeUpdate()
    @BeforeDelete()
    public hook(type: string) {
        // you can update the entity after certain events happened
    }
}

Example: general

async function keyExamples() {
    const connection = await createConnection({keyFilename: "./datastoreServiceAccount.json"});
    const userRepository = connection.getRepository(User);

    const user1 = new User();
    const userKey1 = user1.getKey();
    const userKey2 = userRepository.create().getKey();
    const user3 = userRepository.create({_id: 3});
    user3._ancestorKey = userKey1;
    const userKey3 = user3.getKey();

    const result1 = isSameKey(userKey1, userKey2); // true
    const result2 = isSameNamespace(userKey1, userKey2); // true
    const result3 = isSameKind(userKey1, userKey2); // true
    const result4 = isSamePath(userKey1, userKey2); // true
    const result5 = isSameKey(userKey1, userKey3); // false

    const [encoded1] = await userRepository.datastore.keyToLegacyUrlSafe(userKey1);
    const restoredKey = userRepository.datastore.keyFromLegacyUrlsafe(encoded1);
    const result6 = isSameKey(userKey1, restoredKey); // true
}

Example: general

async function generalExamples() {
    const connection = await createConnection({keyFilename: "./datastoreServiceAccount.json"});
    const repository = connection.getRepository(User, {namespace: "mynamespace", kind: "NewUser"});
    const taskGroupRepository = connection.getRepository(TaskGroup);
    const datastore = connection.datastore; // access to native datastore

    const user1 = repository.create();
    await repository.insert(user1);
    const key = user1.getKey(); // the native datastore key

    // the kind and namespace is attached to entity, but they are not enumerable by default
    // use @Entity({enumerable: true}) such that if you console.log(entity), _kind and _namespace will be displayed as well
    const {_kind, _namespace, _ancestorKey} = user1;

    // simple query
    const findUser1 = repository.findOne(user1._id);

    // find users
    const users = await repository
        .query()
        .filter("_id", x => x.gt(5))
        .limit(100)
        .findMany();

    // get id
    const ids = await repository.allocateIds(10);

    // remove all data
    await repository.truncate();
}

Example: multiple entities

async function multipleEntities() {
    const connection = await createConnection({keyFilename: "./datastoreServiceAccount.json"});
    const repository = connection.getRepository(User, {namespace: "mynamespace", kind: "NewUser"});

    const users = Array(10).map((_, i) => repository.create({number: i}));
    await repository.insert(users);
    await repository.update(users);
    await repository.upsert(users);
    await repository.delete(users);
}

Example: ancestors

async function ancestorExamples() {
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const userRepository = connection.getRepository(User);
    const taskGroupRepository = connection.getRepository(TaskGroup);

    const user1 = await userRepository.insert(userRepository.create({_id: 1}));
    const taskGroup = taskGroupRepository.create({_id: 1, name: "group 1", _ancestorKey: user1.getKey()});
    await taskGroupRepository.insert(taskGroup);

    // ignore the strong type on method call
    const findTaskGroup1 = await taskGroupRepository.query()
        .filterKey(taskGroup.getKey())
        .findOne();

    // get back the user
    if (findTaskGroup1?._ancestorKey) {
        const findUser1 = await userRepository.findOne(findTaskGroup1._ancestorKey);
    }

    // another way to query the entities
    const findTaskGroup2 = await taskGroupRepository.query()
        .setAncestorKey(user1.getKey())
        .filter("_id", 1)
        .findOne();
}

Example: admin

async function adminExamples() {
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const myAdmin = connection.getAdmin();
    const namespaces = await myAdmin.getNamespaces();
    const kinds = await myAdmin.getKinds();
}

Example: query

async function queryExamples() {
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const userRepository = connection.getRepository(User);
    const user = userRepository.create({_id: 1});

    const findUser1 = await userRepository.query().findOne();
    const findUsers2 = await userRepository.findMany([1, 2, 3, 4, 5]);
    const findUsers3 = await userRepository.query().filter("_id", x => x.ge(1).lt(6)).findMany();
    const findUsers4 = await userRepository.query().limit(10).offset(3).order("number", {descending: true}).findMany();

    // complex query with strong type
    const query1 = userRepository.query()
        .filter("number", 10)
        .setAncestorKey(user.getKey())
        .groupBy("number")
        .order("number", {descending: true})
        .offset(5)
        .limit(10);

    // complex query with strong type
    const query = userRepository.query({weakType: true})
        .filter("object.name", 10)
        .setAncestorKey(user.getKey())
        .groupBy("object.name")
        .order("object.name", {descending: true})
        .offset(5)
        .limit(10);

    // iterator
    const batch = 500;
    const iterator = userRepository.query().limit(batch).getAsyncIterator();
    for await (const entities of iterator) {
        if (entities.length === batch) {
            // true
        }
    }

    // select key query
    // this can save some query cost and also return faster
    const keys = await userRepository.selectKeyQuery().findMany();
}

Example: transaction manager

async function transactionManagerExamples() {
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const userRepository = connection.getRepository(User);

    const transactionManager1 = connection.getTransactionManager();

    // customize behavior of the transaction
    // for readonly transaction, please refer to datastore documentation
    const transactionManager2 = connection.getTransactionManager({maxRetry: 3, retryDelay: 200, readOnly: true});

    const result = await transactionManager1.start(async (session) => {
        const findEntity1 = await userRepository.findOneWithSession(1, session);
        const findEntity2 = await userRepository.queryWithSession( session).findOne();
        const ids = await userRepository.allocateIdsWithSession(10, session);

        if (findEntity2) {
            // only the last operation of the same entity will applies only
            userRepository.insertWithSession(findEntity2, session);
            userRepository.updateWithSession(findEntity2, session);
            userRepository.upsertWithSession(findEntity2, session);
            userRepository.deleteWithSession(findEntity2, session);
        } else {
            await session.rollback();
        }

        return 5;
    });

    // value === 5 in above case
    const {value, hasCommitted, totalRetry} = result;
}

Example: error

async function errorExamples() {
    // this help you to provide a better stack upon error
    tsDatastoreOrm.useFriendlyErrorStack = true;

    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const userRepository = connection.getRepository(User);
    const user = userRepository.create({_id: 1});

    try {
        await userRepository.delete(user);
    } catch (err) {
        if (err instanceof TsDatastoreOrmError) {
            // error from this library

        }
    }
}

Example: increment helper

async function incrementHelperExamples() {
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const userRepository = connection.getRepository(User);
    const user = userRepository.create({_id: 1});
    const incrementHelper = userRepository.getIncrementHelper();

    // take all kind of parameters
    const latestValue1 = await incrementHelper.increment(user._id, "number", 10);
    const latestValue2 = await incrementHelper.increment(user, "number", 10);
    const latestValue3 = await incrementHelper.increment(user.getKey(), "number", 10);
}

Example: index resave helper

async function indexResaveHelperExamples() {
    // sometimes u added new index and need to resave the entities
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const userRepository = connection.getRepository(User);
    const helper = userRepository.getIndexResaveHelper();

    await helper.resave("number");
    await helper.resave(["number", "string"]);
}

Example: lock manager

async function lockManagerExamples() {
    // this is a tool for atomic lock
    const connection = await createConnection({clientEmail: "", privateKey: ""});
    const lockManager1 = connection.getLockManager({expiresIn: 1000});

    // this look will try to acquire the lock 2 more times if it failed, waiting 100ms for each retry
    // you can also customize which namespace and kind to save temporary data of the lock
    const lockManager2 = connection.getLockManager({expiresIn: 1000, maxRetry: 2, retryDelay: 100, namespace: "custom", kind: "Lock"});

    try {
        const lockKey = "anyKey";
        const result = await lockManager1.start(lockKey, async () => {
            return 5;
        });
        console.log(result.value);

    } catch (err) {
        // your own error or lock acquire error

    }
}

Example: composite index exporter

async function compositeIndexExamples() {
    const filename = "./index.yaml";
    const exporter = new CompositeIndexExporter();
    exporter.addEntity(User, {kind: "NewUser"});
    exporter.addEntity(TaskGroup, {kind: "NewTaskGroup"});
    // you can add multiple class, but you can't customize the kind name
    exporter.addEntity([User, TaskGroup]);

    const yaml = exporter.getYaml();
    exporter.exportTo(filename);

}

More Examples

Examples are in the tests/ directory.

| Sample | Source Code | | --------------------------- | --------------------------------- | | General | source code | | Query | source code | | Subclass | source code | | Errors | source code | | CompositeIndex | source code | | Admin | source code | | TransactionManager | source code | | LockManager | source code | | Hook | source code | | IndexResaveHelper | source code | | IncrementHelper | source code |

Useful links

• https://googleapis.dev/nodejs/datastore/7.0.0/index.html
• https://googleapis.dev/nodejs/datastore/6.0.0/index.html
• https://cloud.google.com/datastore/docs/
• https://www.npmjs.com/package/@google-cloud/datastore
• https://www.npmjs.com/package/@google-cloud/firestore