diff --git a/.eslintrc.js b/.eslintrc.js deleted file mode 100644 index c83d81a4d..000000000 --- a/.eslintrc.js +++ /dev/null @@ -1,67 +0,0 @@ -// The ESLint ecmaVersion argument is inconsistently used. Some rules will ignore it entirely, so if the rule has -// been set, it will still error even if it's not applicable to that version number. Since Google sets these -// rules, we have to turn them off ourselves. -var DISABLED_ES6_OPTIONS = { - 'no-var': 'off', - 'prefer-rest-params': 'off' -}; - -var SHAREDB_RULES = { - // Comma dangle is not supported in ES3 - 'comma-dangle': ['error', 'never'], - // We control our own objects and prototypes, so no need for this check - 'guard-for-in': 'off', - // Google prescribes different indents for different cases. Let's just use 2 spaces everywhere. Note that we have - // to override ESLint's default of 0 indents for this. - indent: ['error', 2, { - SwitchCase: 1 - }], - // Less aggressive line length than Google, which is especially useful when we have a lot of callbacks in our code - 'max-len': ['error', - { - code: 120, - tabWidth: 2, - ignoreUrls: true - } - ], - // Google overrides the default ESLint behaviour here, which is slightly better for catching erroneously unused - // variables - 'no-unused-vars': ['error', {vars: 'all', args: 'after-used'}], - // It's more readable to ensure we only have one statement per line - 'max-statements-per-line': ['error', {max: 1}], - // ES3 doesn't support spread - 'prefer-spread': 'off', - // as-needed quote props are easier to write - 'quote-props': ['error', 'as-needed'], - 'require-jsdoc': 'off', - 'valid-jsdoc': 'off' -}; - -module.exports = { - extends: 'google', - parserOptions: { - ecmaVersion: 3, - allowReserved: true - }, - rules: Object.assign( - {}, - DISABLED_ES6_OPTIONS, - SHAREDB_RULES - ), - ignorePatterns: [ - '/docs/' - ], - overrides: [ - { - files: ['examples/counter-json1-vite/*.js'], - parserOptions: { - ecmaVersion: 6, - sourceType: 'module', - allowReserved: false - }, - rules: { - quotes: ['error', 'single'] - } - } - ] -}; diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f0bea60da..2f9417ebf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,9 +4,11 @@ on: push: branches: - master + - typescript pull_request: branches: - master + - typescript jobs: test: @@ -15,9 +17,9 @@ jobs: strategy: matrix: node: - - 18 - 20 - 22 + - 24 services: mongodb: image: mongo:4.4 @@ -32,6 +34,7 @@ jobs: - name: Install run: npm install - name: Lint + if: github.ref_name != 'typescript' && github.base_ref != 'typescript' # Remove this condition after TS code lints cleanly run: npm run lint - name: Test run: npm run test-cover diff --git a/.gitignore b/.gitignore index 1086994ce..fae7ca800 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,9 @@ node_modules package-lock.json jspm_packages +# TypeScript output +lib/ + # Don't commit generated JS bundles examples/**/static/dist/bundle.js examples/**/dist diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 000000000..a2d031e01 --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,96 @@ +const jsEslint = require("@eslint/js"); +const { + defineConfig, + globalIgnores, +} = require("eslint/config"); +const eslintConfigGoogle = require('eslint-config-google'); +const tsEslint = require('typescript-eslint'); + +// The ESLint ecmaVersion argument is inconsistently used. Some rules will ignore it entirely, so if the rule has +// been set, it will still error even if it's not applicable to that version number. Since Google sets these +// rules, we have to turn them off ourselves. +var DISABLED_ES6_OPTIONS = { + 'no-var': 'off', + 'prefer-rest-params': 'off' +}; + +var SHAREDB_RULES = { + // Comma dangle is not supported in ES3 + 'comma-dangle': ['error', 'never'], + // We control our own objects and prototypes, so no need for this check + 'guard-for-in': 'off', + // Google prescribes different indents for different cases. Let's just use 2 spaces everywhere. Note that we have + // to override ESLint's default of 0 indents for this. + indent: ['error', 2, { + SwitchCase: 1 + }], + 'linebreak-style': 'off', + // Less aggressive line length than Google, which is especially useful when we have a lot of callbacks in our code + 'max-len': ['error', + { + code: 120, + tabWidth: 2, + ignoreUrls: true + } + ], + // Google overrides the default ESLint behaviour here, which is slightly better for catching erroneously unused + // variables + 'no-unused-vars': ['error', { + vars: 'all', + args: 'after-used', + // This can be removed once the minimum ES version is ES2019 or newer, and catch statements + // are updated to use optional catch binding. + caughtErrors: 'none', + }], + // It's more readable to ensure we only have one statement per line + 'max-statements-per-line': ['error', { max: 1 }], + // ES3 doesn't support spread + 'prefer-spread': 'off', + // as-needed quote props are easier to write + 'quote-props': ['error', 'as-needed'], + 'require-jsdoc': 'off', + 'valid-jsdoc': 'off', +}; + +module.exports = defineConfig([ + { + extends: [eslintConfigGoogle], + files: ["**/*.js"], + ignores: ['eslint.config.js'], + + languageOptions: { + ecmaVersion: 3, + sourceType: "commonjs", + + parserOptions: { + allowReserved: true, + }, + }, + + rules: Object.assign({}, DISABLED_ES6_OPTIONS, SHAREDB_RULES), + }, + { + extends: [ + jsEslint.configs.recommended, + tsEslint.configs.recommended, + ], + files: ["**/*.ts"], + }, + globalIgnores(["docs/"]), + { + files: ["examples/counter-json1-vite/*.js"], + + languageOptions: { + ecmaVersion: 2015, + sourceType: "module", + + parserOptions: { + allowReserved: false, + }, + }, + + rules: { + quotes: ["error", "single"], + }, + } +]); diff --git a/lib/agent.js b/lib/agent.js deleted file mode 100644 index a07650c8a..000000000 --- a/lib/agent.js +++ /dev/null @@ -1,1035 +0,0 @@ -var hat = require('hat'); -var ShareDBError = require('./error'); -var logger = require('./logger'); -var ACTIONS = require('./message-actions').ACTIONS; -var types = require('./types'); -var util = require('./util'); -var protocol = require('./protocol'); - -var ERROR_CODE = ShareDBError.CODES; - -/** - * Agent deserializes the wire protocol messages received from the stream and - * calls the corresponding functions on its Agent. It uses the return values - * to send responses back. Agent also handles piping the operation streams - * provided by a Agent. - * - * @param {Backend} backend - * @param {Duplex} stream connection to a client - */ -function Agent(backend, stream) { - this.backend = backend; - this.stream = stream; - - this.clientId = hat(); - // src is a client-configurable "id" which the client will set in its handshake, - // and attach to its ops. This should take precedence over clientId if set. - // Only legacy clients, or new clients connecting for the first time will use the - // Agent-provided clientId. Ideally we'll deprecate clientId in favour of src - // in the next breaking change. - this.src = null; - this.connectTime = Date.now(); - - // We need to track which documents are subscribed by the client. This is a - // map of collection -> id -> stream - this.subscribedDocs = Object.create(null); - - // Map from queryId -> emitter - this.subscribedQueries = Object.create(null); - - // Track which documents are subscribed to presence by the client. This is a - // map of channel -> stream - this.subscribedPresences = Object.create(null); - // Highest seq received for a subscription request. Any seq lower than this - // value is stale, and should be ignored. Used for keeping the subscription - // state in sync with the client's desired state. Map of channel -> seq - this.presenceSubscriptionSeq = Object.create(null); - // Keep track of the last request that has been sent by each local presence - // belonging to this agent. This is used to generate a new disconnection - // request if the client disconnects ungracefully. This is a - // map of channel -> id -> request - this.presenceRequests = Object.create(null); - // Keep track of the latest known Doc version, so that we can avoid fetching - // ops to transform presence if not needed - this.latestDocVersionStreams = Object.create(null); - this.latestDocVersions = Object.create(null); - - // We need to track this manually to make sure we don't reply to messages - // after the stream was closed. - this.closed = false; - - // For custom use in middleware. The agent is a convenient place to cache - // session state in memory. It is in memory only as long as the session is - // active, and it is passed to each middleware call - this.custom = Object.create(null); - - this.protocol = Object.create(null); - - // The first message received over the connection. Stored to warn if messages - // are being sent before the handshake. - this._firstReceivedMessage = null; - this._handshakeReceived = false; - - // Send the legacy message to initialize old clients with the random agent Id - this.send(this._initMessage(ACTIONS.initLegacy)); -} -module.exports = Agent; - -// Close the agent with the client. -Agent.prototype.close = function(err) { - if (err) { - logger.warn('Agent closed due to error', this._src(), err.stack || err); - } - if (this.closed) return; - // This will end the writable stream and emit 'finish' - this.stream.end(); -}; - -Agent.prototype._cleanup = function() { - // Only clean up once if the stream emits both 'end' and 'close'. - if (this.closed) return; - - this.closed = true; - - this.backend.agentsCount--; - if (!this.stream.isServer) this.backend.remoteAgentsCount--; - - // Clean up doc subscription streams - for (var collection in this.subscribedDocs) { - var docs = this.subscribedDocs[collection]; - for (var id in docs) { - var stream = docs[id]; - stream.destroy(); - } - } - this.subscribedDocs = Object.create(null); - - for (var channel in this.subscribedPresences) { - this.subscribedPresences[channel].destroy(); - } - this.subscribedPresences = Object.create(null); - - // Clean up query subscription streams - for (var id in this.subscribedQueries) { - var emitter = this.subscribedQueries[id]; - emitter.destroy(); - } - this.subscribedQueries = Object.create(null); - - for (var collection in this.latestDocVersionStreams) { - var streams = this.latestDocVersionStreams[collection]; - for (var id in streams) streams[id].destroy(); - } - this.latestDocVersionStreams = Object.create(null); -}; - -/** - * Passes operation data received on stream to the agent stream via - * _sendOp() - */ -Agent.prototype._subscribeToStream = function(collection, id, stream) { - var agent = this; - this._subscribeMapToStream(this.subscribedDocs, collection, id, stream, function(data) { - if (data.error) { - // Log then silently ignore errors in a subscription stream, since these - // may not be the client's fault, and they were not the result of a - // direct request by the client - logger.error('Doc subscription stream error', collection, id, data.error); - return; - } - agent._onOp(collection, id, data); - }); -}; - -Agent.prototype._subscribeMapToStream = function(map, collection, id, stream, dataHandler) { - if (this.closed) return stream.destroy(); - - var streams = map[collection] || (map[collection] = Object.create(null)); - - // If already subscribed to this document, destroy the previously subscribed stream - var previous = streams[id]; - if (previous) previous.destroy(); - streams[id] = stream; - - stream.on('data', dataHandler); - stream.on('end', function() { - // The op stream is done sending, so release its reference - var streams = map[collection]; - if (!streams || streams[id] !== stream) return; - delete streams[id]; - if (util.hasKeys(streams)) return; - delete map[collection]; - }); -}; - -Agent.prototype._subscribeToPresenceStream = function(channel, stream) { - if (this.closed) return stream.destroy(); - var agent = this; - - stream.on('data', function(data) { - if (data.error) { - logger.error('Presence subscription stream error', channel, data.error); - } - agent._handlePresenceData(data); - }); - - stream.on('end', function() { - var requests = agent.presenceRequests[channel] || {}; - for (var id in requests) { - var request = agent.presenceRequests[channel][id]; - request.seq++; - request.p = null; - agent._broadcastPresence(request, function(error) { - if (error) logger.error('Error broadcasting disconnect presence', channel, error); - }); - } - if (agent.subscribedPresences[channel] === stream) { - delete agent.subscribedPresences[channel]; - } - delete agent.presenceRequests[channel]; - }); -}; - -Agent.prototype._subscribeToQuery = function(emitter, queryId, collection, query) { - var previous = this.subscribedQueries[queryId]; - if (previous) previous.destroy(); - this.subscribedQueries[queryId] = emitter; - - var agent = this; - emitter.onExtra = function(extra) { - agent.send({a: ACTIONS.queryUpdate, id: queryId, extra: extra}); - }; - - emitter.onDiff = function(diff) { - for (var i = 0; i < diff.length; i++) { - var item = diff[i]; - if (item.type === 'insert') { - item.values = getResultsData(item.values); - } - } - // Consider stripping the collection out of the data we send here - // if it matches the query's collection. - agent.send({a: ACTIONS.queryUpdate, id: queryId, diff: diff}); - }; - - emitter.onError = function(err) { - // Log then silently ignore errors in a subscription stream, since these - // may not be the client's fault, and they were not the result of a - // direct request by the client - logger.error('Query subscription stream error', collection, query, err); - }; - - emitter.onOp = function(op) { - var id = op.d; - agent._onOp(collection, id, op); - }; - - emitter._open(); -}; - -Agent.prototype._onOp = function(collection, id, op) { - if (this._isOwnOp(collection, op)) return; - - // Ops emitted here are coming directly from pubsub, which emits the same op - // object to listeners without making a copy. The pattern in middleware is to - // manipulate the passed in object, and projections are implemented the same - // way currently. - // - // Deep copying the op would be safest, but deep copies are very expensive, - // especially over arbitrary objects. This function makes a shallow copy of an - // op, and it requires that projections and any user middleware copy deep - // properties as needed when they modify the op. - // - // Polling of query subscriptions is determined by the same op objects. As a - // precaution against op middleware breaking query subscriptions, we delay - // before calling into projection and middleware code - var agent = this; - util.nextTick(function() { - var copy = shallowCopy(op); - agent.backend.sanitizeOp(agent, collection, id, copy, function(err) { - if (err) { - logger.error('Error sanitizing op emitted from subscription', collection, id, copy, err); - return; - } - agent._sendOp(collection, id, copy); - }); - }); -}; - -Agent.prototype._isOwnOp = function(collection, op) { - // Detect ops from this client on the same projection. Since the client sent - // these in, the submit reply will be sufficient and we can silently ignore - // them in the streams for subscribed documents or queries - return (this._src() === op.src) && (collection === (op.i || op.c)); -}; - -Agent.prototype.send = function(message) { - // Quietly drop replies if the stream was closed - if (this.closed) return; - - this.backend.emit('send', this, message); - this.stream.write(message); -}; - -Agent.prototype._sendOp = function(collection, id, op) { - var message = { - a: ACTIONS.op, - c: collection, - d: id, - v: op.v, - src: op.src, - seq: op.seq - }; - if ('op' in op) message.op = op.op; - if (op.create) message.create = op.create; - if (op.del) message.del = true; - - this.send(message); -}; -Agent.prototype._sendOps = function(collection, id, ops) { - for (var i = 0; i < ops.length; i++) { - this._sendOp(collection, id, ops[i]); - } -}; -Agent.prototype._sendOpsBulk = function(collection, opsMap) { - for (var id in opsMap) { - var ops = opsMap[id]; - this._sendOps(collection, id, ops); - } -}; - -function getReplyErrorObject(err) { - if (typeof err === 'string') { - return { - code: ERROR_CODE.ERR_UNKNOWN_ERROR, - message: err - }; - } else { - if (err.stack) { - logger.info(err.stack); - } - return { - code: err.code, - message: err.message - }; - } -} - -Agent.prototype._reply = function(request, err, message) { - var agent = this; - var backend = agent.backend; - if (err) { - request.error = getReplyErrorObject(err); - agent.send(request); - return; - } - if (!message) message = {}; - - message.a = request.a; - if (request.id) { - message.id = request.id; - } else { - if (request.c) message.c = request.c; - if (request.d) message.d = request.d; - if (request.b && !message.data) message.b = request.b; - } - - var middlewareContext = {request: request, reply: message}; - backend.trigger(backend.MIDDLEWARE_ACTIONS.reply, agent, middlewareContext, function(err) { - if (err) { - request.error = getReplyErrorObject(err); - agent.send(request); - } else { - agent.send(middlewareContext.reply); - } - }); -}; - -// Start processing events from the stream -Agent.prototype._open = function() { - if (this.closed) return; - this.backend.agentsCount++; - if (!this.stream.isServer) this.backend.remoteAgentsCount++; - - var agent = this; - this.stream.on('data', function(chunk) { - if (agent.closed) return; - - if (typeof chunk !== 'object') { - var err = new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Received non-object message'); - return agent.close(err); - } - - var request = {data: chunk}; - agent.backend.trigger(agent.backend.MIDDLEWARE_ACTIONS.receive, agent, request, function(err) { - var callback = function(err, message) { - agent._reply(request.data, err, message); - }; - if (err) return callback(err); - agent._handleMessage(request.data, callback); - }); - }); - - var cleanup = agent._cleanup.bind(agent); - this.stream.on('end', cleanup); - this.stream.on('close', cleanup); -}; - -// Check a request to see if its valid. Returns an error if there's a problem. -Agent.prototype._checkRequest = function(request) { - if ( - request.a === ACTIONS.queryFetch || - request.a === ACTIONS.querySubscribe || - request.a === ACTIONS.queryUnsubscribe - ) { - // Query messages need an ID property. - if (typeof request.id !== 'number') return 'Missing query ID'; - } else if (request.a === ACTIONS.op || - request.a === ACTIONS.fetch || - request.a === ACTIONS.subscribe || - request.a === ACTIONS.unsubscribe || - request.a === ACTIONS.presence) { - // Doc-based request. - if (request.c != null) { - if (typeof request.c !== 'string' || util.isDangerousProperty(request.c)) { - return 'Invalid collection'; - } - } - if (request.d != null) { - if (typeof request.d !== 'string' || util.isDangerousProperty(request.d)) { - return 'Invalid id'; - } - } - - if (request.a === ACTIONS.op || request.a === ACTIONS.presence) { - if (request.v != null && (typeof request.v !== 'number' || request.v < 0)) return 'Invalid version'; - } - - if (request.a === ACTIONS.presence) { - if (typeof request.id !== 'string' || util.isDangerousProperty(request.id)) { - return 'Invalid presence ID'; - } - } - } else if ( - request.a === ACTIONS.bulkFetch || - request.a === ACTIONS.bulkSubscribe || - request.a === ACTIONS.bulkUnsubscribe - ) { - // Bulk request - if (request.c != null) { - if (typeof request.c !== 'string' || util.isDangerousProperty(request.c)) { - return 'Invalid collection'; - } - } - if (typeof request.b !== 'object') return 'Invalid bulk subscribe data'; - } - if (request.ch != null) { - if (typeof request.ch !== 'string' || util.isDangerousProperty(request.ch)) { - return 'Invalid presence channel'; - } - } -}; - -// Handle an incoming message from the client -Agent.prototype._handleMessage = function(request, callback) { - try { - var errMessage = this._checkRequest(request); - if (errMessage) return callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, errMessage)); - this._checkFirstMessage(request); - - switch (request.a) { - case ACTIONS.handshake: - if (request.id) this.src = request.id; - this._setProtocol(request); - return callback(null, this._initMessage(ACTIONS.handshake)); - case ACTIONS.queryFetch: - return this._queryFetch(request.id, request.c, request.q, getQueryOptions(request), callback); - case ACTIONS.querySubscribe: - return this._querySubscribe(request.id, request.c, request.q, getQueryOptions(request), callback); - case ACTIONS.queryUnsubscribe: - return this._queryUnsubscribe(request.id, callback); - case ACTIONS.bulkFetch: - return this._fetchBulk(request.c, request.b, callback); - case ACTIONS.bulkSubscribe: - return this._subscribeBulk(request.c, request.b, callback); - case ACTIONS.bulkUnsubscribe: - return this._unsubscribeBulk(request.c, request.b, callback); - case ACTIONS.fetch: - return this._fetch(request.c, request.d, request.v, callback); - case ACTIONS.subscribe: - return this._subscribe(request.c, request.d, request.v, callback); - case ACTIONS.unsubscribe: - return this._unsubscribe(request.c, request.d, callback); - case ACTIONS.op: - // Normalize the properties submitted - var op = createClientOp(request, this._src()); - if (op.seq >= util.MAX_SAFE_INTEGER) { - return callback(new ShareDBError( - ERROR_CODE.ERR_CONNECTION_SEQ_INTEGER_OVERFLOW, - 'Connection seq has exceeded the max safe integer, maybe from being open for too long' - )); - } - if (!op) return callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Invalid op message')); - return this._submit(request.c, request.d, op, callback); - case ACTIONS.snapshotFetch: - return this._fetchSnapshot(request.c, request.d, request.v, callback); - case ACTIONS.snapshotFetchByTimestamp: - return this._fetchSnapshotByTimestamp(request.c, request.d, request.ts, callback); - case ACTIONS.presence: - if (!this.backend.presenceEnabled) return; - var presence = this._createPresence(request); - if (presence.t && !util.supportsPresence(types.map[presence.t])) { - return callback({ - code: ERROR_CODE.ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE, - message: 'Type does not support presence: ' + presence.t - }); - } - return this._broadcastPresence(presence, callback); - case ACTIONS.presenceSubscribe: - if (!this.backend.presenceEnabled) return; - return this._subscribePresence(request.ch, request.seq, callback); - case ACTIONS.presenceUnsubscribe: - return this._unsubscribePresence(request.ch, request.seq, callback); - case ACTIONS.presenceRequest: - return this._requestPresence(request.ch, callback); - case ACTIONS.pingPong: - return this._pingPong(callback); - default: - callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Invalid or unknown message')); - } - } catch (err) { - callback(err); - } -}; -function getQueryOptions(request) { - var results = request.r; - var ids; - var fetch; - var fetchOps; - if (results) { - ids = []; - for (var i = 0; i < results.length; i++) { - var result = results[i]; - var id = result[0]; - var version = result[1]; - ids.push(id); - if (version == null) { - if (fetch) { - fetch.push(id); - } else { - fetch = [id]; - } - } else { - if (!fetchOps) fetchOps = Object.create(null); - fetchOps[id] = version; - } - } - } - var options = request.o || {}; - options.ids = ids; - options.fetch = fetch; - options.fetchOps = fetchOps; - return options; -} - -Agent.prototype._queryFetch = function(queryId, collection, query, options, callback) { - // Fetch the results of a query once - this.backend.queryFetch(this, collection, query, options, function(err, results, extra) { - if (err) return callback(err); - var message = { - data: getResultsData(results), - extra: extra - }; - callback(null, message); - }); -}; - -Agent.prototype._querySubscribe = function(queryId, collection, query, options, callback) { - // Subscribe to a query. The client is sent the query results and its - // notified whenever there's a change - var agent = this; - var wait = 1; - var message; - function finish(err) { - if (err) return callback(err); - if (--wait) return; - callback(null, message); - } - if (options.fetch) { - wait++; - this.backend.fetchBulk(this, collection, options.fetch, function(err, snapshotMap) { - if (err) return finish(err); - message = getMapResult(snapshotMap); - finish(); - }); - } - if (options.fetchOps) { - wait++; - this._fetchBulkOps(collection, options.fetchOps, finish); - } - this.backend.querySubscribe(this, collection, query, options, function(err, emitter, results, extra) { - if (err) return finish(err); - if (agent.closed) return emitter.destroy(); - - agent._subscribeToQuery(emitter, queryId, collection, query); - // No results are returned when ids are passed in as an option. Instead, - // want to re-poll the entire query once we've established listeners to - // emit any diff in results - if (!results) { - emitter.queryPoll(finish); - return; - } - message = { - data: getResultsData(results), - extra: extra - }; - finish(); - }); -}; - -Agent.prototype._pingPong = function(callback) { - var error = null; - var message = { - a: ACTIONS.pingPong - }; - callback(error, message); -}; - -function getResultsData(results) { - var items = []; - for (var i = 0; i < results.length; i++) { - var result = results[i]; - var item = getSnapshotData(result); - item.d = result.id; - items.push(item); - } - return items; -} - -function getMapResult(snapshotMap) { - var data = Object.create(null); - for (var id in snapshotMap) { - var mapValue = snapshotMap[id]; - // fetchBulk / subscribeBulk map data can have either a Snapshot or an object - // `{error: Error | string}` as a value. - if (mapValue.error) { - // Transform errors to serialization-friendly objects. - data[id] = {error: getReplyErrorObject(mapValue.error)}; - } else { - data[id] = getSnapshotData(mapValue); - } - } - return {data: data}; -} - -function getSnapshotData(snapshot) { - var data = { - v: snapshot.v, - data: snapshot.data - }; - if (types.defaultType !== types.map[snapshot.type]) { - data.type = snapshot.type; - } - return data; -} - -Agent.prototype._queryUnsubscribe = function(queryId, callback) { - var emitter = this.subscribedQueries[queryId]; - if (emitter) { - emitter.destroy(); - delete this.subscribedQueries[queryId]; - } - util.nextTick(callback); -}; - -Agent.prototype._fetch = function(collection, id, version, callback) { - if (version == null) { - // Fetch a snapshot - this.backend.fetch(this, collection, id, function(err, snapshot) { - if (err) return callback(err); - callback(null, {data: getSnapshotData(snapshot)}); - }); - } else { - // It says fetch on the tin, but if a version is specified the client - // actually wants me to fetch some ops - this._fetchOps(collection, id, version, callback); - } -}; - -Agent.prototype._fetchOps = function(collection, id, version, callback) { - var agent = this; - this.backend.getOps(this, collection, id, version, null, function(err, ops) { - if (err) return callback(err); - agent._sendOps(collection, id, ops); - callback(); - }); -}; - -Agent.prototype._fetchBulk = function(collection, versions, callback) { - if (Array.isArray(versions)) { - this.backend.fetchBulk(this, collection, versions, function(err, snapshotMap) { - if (err) { - return callback(err); - } - if (snapshotMap) { - var result = getMapResult(snapshotMap); - callback(null, result); - } else { - callback(); - } - }); - } else { - this._fetchBulkOps(collection, versions, callback); - } -}; - -Agent.prototype._fetchBulkOps = function(collection, versions, callback) { - var agent = this; - this.backend.getOpsBulk(this, collection, versions, null, function(err, opsMap) { - if (err) return callback(err); - agent._sendOpsBulk(collection, opsMap); - callback(); - }); -}; - -Agent.prototype._subscribe = function(collection, id, version, callback) { - // If the version is specified, catch the client up by sending all ops - // since the specified version - var agent = this; - this.backend.subscribe(this, collection, id, version, function(err, stream, snapshot, ops) { - if (err) return callback(err); - // If we're subscribing from a known version, send any ops committed since - // the requested version to bring the client's doc up to date - if (ops) { - agent._sendOps(collection, id, ops); - } - // In addition, ops may already be queued on the stream by pubsub. - // Subscribe is called before the ops or snapshot are fetched, so it is - // possible that some ops may be duplicates. Clients should ignore any - // duplicate ops they may receive. This will flush ops already queued and - // subscribe to ongoing ops from the stream - agent._subscribeToStream(collection, id, stream); - // Snapshot is returned only when subscribing from a null version. - // Otherwise, ops will have been pushed into the stream - if (snapshot) { - callback(null, {data: getSnapshotData(snapshot)}); - } else { - callback(); - } - }); -}; - -Agent.prototype._subscribeBulk = function(collection, versions, callback) { - // See _subscribe() above. This function's logic should match but in bulk - var agent = this; - this.backend.subscribeBulk(this, collection, versions, function(err, streams, snapshotMap, opsMap) { - if (err) { - return callback(err); - } - if (opsMap) { - agent._sendOpsBulk(collection, opsMap); - } - for (var id in streams) { - agent._subscribeToStream(collection, id, streams[id]); - } - if (snapshotMap) { - var result = getMapResult(snapshotMap); - callback(null, result); - } else { - callback(); - } - }); -}; - -Agent.prototype._unsubscribe = function(collection, id, callback) { - // Unsubscribe from the specified document. This cancels the active - // stream or an inflight subscribing state - var docs = this.subscribedDocs[collection]; - var stream = docs && docs[id]; - if (stream) stream.destroy(); - util.nextTick(callback); -}; - -Agent.prototype._unsubscribeBulk = function(collection, ids, callback) { - var docs = this.subscribedDocs[collection]; - if (!docs) return util.nextTick(callback); - for (var i = 0; i < ids.length; i++) { - var id = ids[i]; - var stream = docs[id]; - if (stream) stream.destroy(); - } - util.nextTick(callback); -}; - -Agent.prototype._submit = function(collection, id, op, callback) { - var agent = this; - this.backend.submit(this, collection, id, op, null, function(err, ops, request) { - // Message to acknowledge the op was successfully submitted - var ack = {src: op.src, seq: op.seq, v: op.v}; - if (request._fixupOps.length) ack[ACTIONS.fixup] = request._fixupOps; - if (err) { - // Occasional 'Op already submitted' errors are expected to happen as - // part of normal operation, since inflight ops need to be resent after - // disconnect. In this case, ack the op so the client can proceed - if (err.code === ERROR_CODE.ERR_OP_ALREADY_SUBMITTED) return callback(null, ack); - return callback(err); - } - - // Reply with any operations that the client is missing. - agent._sendOps(collection, id, ops); - callback(null, ack); - }); -}; - -Agent.prototype._fetchSnapshot = function(collection, id, version, callback) { - this.backend.fetchSnapshot(this, collection, id, version, callback); -}; - -Agent.prototype._fetchSnapshotByTimestamp = function(collection, id, timestamp, callback) { - this.backend.fetchSnapshotByTimestamp(this, collection, id, timestamp, callback); -}; - -Agent.prototype._initMessage = function(action) { - return { - a: action, - protocol: protocol.major, - protocolMinor: protocol.minor, - id: this._src(), - type: types.defaultType.uri - }; -}; - -Agent.prototype._src = function() { - return this.src || this.clientId; -}; - -Agent.prototype._broadcastPresence = function(presence, callback) { - var agent = this; - var backend = this.backend; - var presenceRequests = this.presenceRequests; - var context = { - presence: presence, - collection: presence.c - }; - var start = Date.now(); - - var subscriptionUpdater = presence.p === null ? - this._unsubscribeDocVersion.bind(this) : - this._subscribeDocVersion.bind(this); - - subscriptionUpdater(presence.c, presence.d, function(error) { - if (error) return callback(error); - backend.trigger(backend.MIDDLEWARE_ACTIONS.receivePresence, agent, context, function(error) { - if (error) return callback(error); - var requests = presenceRequests[presence.ch] || (presenceRequests[presence.ch] = Object.create(null)); - var previousRequest = requests[presence.id]; - if (!previousRequest || previousRequest.pv < presence.pv) { - presenceRequests[presence.ch][presence.id] = presence; - } - - var transformer = function(agent, presence, callback) { - callback(null, presence); - }; - - var latestDocVersion = util.dig(agent.latestDocVersions, presence.c, presence.d); - var presenceIsUpToDate = presence.v === latestDocVersion; - if (!presenceIsUpToDate) { - transformer = backend.transformPresenceToLatestVersion.bind(backend); - } - - transformer(agent, presence, function(error, presence) { - if (error) return callback(error); - var channel = agent._getPresenceChannel(presence.ch); - agent.backend.pubsub.publish([channel], presence, function(error) { - if (error) return callback(error); - backend.emit('timing', 'presence.broadcast', Date.now() - start, context); - callback(null, presence); - }); - }); - }); - }); -}; - -Agent.prototype._subscribeDocVersion = function(collection, id, callback) { - if (!collection || !id) return callback(); - - var latestDocVersions = this.latestDocVersions; - var isSubscribed = util.dig(latestDocVersions, collection, id) !== undefined; - if (isSubscribed) return callback(); - - var agent = this; - this.backend.subscribe(this, collection, id, null, function(error, stream, snapshot) { - if (error) return callback(error); - - var versions = latestDocVersions[collection] || (latestDocVersions[collection] = Object.create(null)); - versions[id] = snapshot.v; - - agent._subscribeMapToStream(agent.latestDocVersionStreams, collection, id, stream, function(op) { - // op.v behind snapshot.v by 1 - latestDocVersions[collection][id] = op.v + 1; - }); - - callback(); - }); -}; - -Agent.prototype._unsubscribeDocVersion = function(collection, id, callback) { - var stream = util.dig(this.latestDocVersionStreams, collection, id); - if (stream) stream.destroy(); - util.digAndRemove(this.latestDocVersions, collection, id); - util.nextTick(callback); -}; - -Agent.prototype._createPresence = function(request) { - return { - a: ACTIONS.presence, - ch: request.ch, - src: this._src(), - id: request.id, // Presence ID, not Doc ID (which is 'd') - p: request.p, - pv: request.pv, - // The c,d,v,t fields are only set for DocPresence - c: request.c, - d: request.d, - v: request.v, - t: request.t - }; -}; - -Agent.prototype._subscribePresence = function(channel, seq, cb) { - var agent = this; - - function callback(error) { - cb(error, {ch: channel, seq: seq}); - } - - var existingStream = agent.subscribedPresences[channel]; - if (existingStream) { - agent.presenceSubscriptionSeq[channel] = seq; - return callback(); - } - - var presenceChannel = this._getPresenceChannel(channel); - this.backend.pubsub.subscribe(presenceChannel, function(error, stream) { - if (error) return callback(error); - if (seq < agent.presenceSubscriptionSeq[channel]) { - stream.destroy(); - return callback(); - } - agent.presenceSubscriptionSeq[channel] = seq; - agent.subscribedPresences[channel] = stream; - agent._subscribeToPresenceStream(channel, stream); - agent._requestPresence(channel, function(error) { - callback(error); - }); - }); -}; - -Agent.prototype._unsubscribePresence = function(channel, seq, callback) { - this.presenceSubscriptionSeq[channel] = seq; - var stream = this.subscribedPresences[channel]; - if (stream) stream.destroy(); - delete this.subscribedPresences[channel]; - callback(null, {ch: channel, seq: seq}); -}; - -Agent.prototype._getPresenceChannel = function(channel) { - return '$presence.' + channel; -}; - -Agent.prototype._requestPresence = function(channel, callback) { - var presenceChannel = this._getPresenceChannel(channel); - this.backend.pubsub.publish([presenceChannel], {ch: channel, r: true, src: this.clientId}, callback); -}; - -Agent.prototype._handlePresenceData = function(presence) { - if (presence.src === this._src()) return; - - if (presence.r) return this.send({a: ACTIONS.presenceRequest, ch: presence.ch}); - - var backend = this.backend; - var context = { - collection: presence.c, - presence: presence - }; - var agent = this; - backend.trigger(backend.MIDDLEWARE_ACTIONS.sendPresence, this, context, function(error) { - if (error) { - if (backend.doNotForwardSendPresenceErrorsToClient) backend.errorHandler(error, {agent: agent}); - else agent.send({a: ACTIONS.presence, ch: presence.ch, id: presence.id, error: getReplyErrorObject(error)}); - return; - } - agent.send(presence); - }); -}; - -Agent.prototype._checkFirstMessage = function(request) { - if (this._handshakeReceived) return; - if (!this._firstReceivedMessage) this._firstReceivedMessage = request; - - if (request.a === ACTIONS.handshake) { - this._handshakeReceived = true; - if (this._firstReceivedMessage.a !== ACTIONS.handshake) { - logger.warn('Unexpected message received before handshake', this._firstReceivedMessage); - } - // Release memory - this._firstReceivedMessage = null; - } -}; - -Agent.prototype._setProtocol = function(request) { - this.protocol.major = request.protocol; - this.protocol.minor = request.protocolMinor; -}; - -function createClientOp(request, clientId) { - // src can be provided if it is not the same as the current agent, - // such as a resubmission after a reconnect, but it usually isn't needed - var src = request.src || clientId; - // c, d, and m arguments are intentionally undefined. These are set later - return ('op' in request) ? new EditOp(src, request.seq, request.v, request.op, request.x) : - (request.create) ? new CreateOp(src, request.seq, request.v, request.create, request.x) : - (request.del) ? new DeleteOp(src, request.seq, request.v, request.del, request.x) : - undefined; -} - -function shallowCopy(object) { - var out = {}; - for (var key in object) { - if (util.hasOwn(object, key)) { - out[key] = object[key]; - } - } - return out; -} - -function CreateOp(src, seq, v, create, x, c, d, m) { - this.src = src; - this.seq = seq; - this.v = v; - this.create = create; - this.c = c; - this.d = d; - this.m = m; - this.x = x; -} -function EditOp(src, seq, v, op, x, c, d, m) { - this.src = src; - this.seq = seq; - this.v = v; - this.op = op; - this.c = c; - this.d = d; - this.m = m; - this.x = x; -} -function DeleteOp(src, seq, v, del, x, c, d, m) { - this.src = src; - this.seq = seq; - this.v = v; - this.del = del; - this.c = c; - this.d = d; - this.m = m; - this.x = x; -} diff --git a/lib/backend.js b/lib/backend.js deleted file mode 100644 index dcc95aa33..000000000 --- a/lib/backend.js +++ /dev/null @@ -1,951 +0,0 @@ -var async = require('async'); -var Agent = require('./agent'); -var Connection = require('./client/connection'); -var emitter = require('./emitter'); -var MemoryDB = require('./db/memory'); -var NoOpMilestoneDB = require('./milestone-db/no-op'); -var MemoryPubSub = require('./pubsub/memory'); -var ot = require('./ot'); -var projections = require('./projections'); -var QueryEmitter = require('./query-emitter'); -var ShareDBError = require('./error'); -var Snapshot = require('./snapshot'); -var StreamSocket = require('./stream-socket'); -var SubmitRequest = require('./submit-request'); -var ReadSnapshotsRequest = require('./read-snapshots-request'); -var util = require('./util'); -var logger = require('./logger'); - -var ERROR_CODE = ShareDBError.CODES; - -function Backend(options) { - if (!(this instanceof Backend)) return new Backend(options); - emitter.EventEmitter.call(this); - - if (!options) options = {}; - this.db = options.db || new MemoryDB(); - this.pubsub = options.pubsub || new MemoryPubSub(); - // This contains any extra databases that can be queried - this.extraDbs = options.extraDbs || {}; - this.milestoneDb = options.milestoneDb || new NoOpMilestoneDB(); - - // Map from projected collection -> {type, fields} - this.projections = Object.create(null); - - this.suppressPublish = !!options.suppressPublish; - this.maxSubmitRetries = options.maxSubmitRetries || null; - this.presenceEnabled = !!options.presence; - this.doNotForwardSendPresenceErrorsToClient = !!options.doNotForwardSendPresenceErrorsToClient; - if (this.presenceEnabled && !this.doNotForwardSendPresenceErrorsToClient) { - logger.warn( - 'Broadcasting "sendPresence" middleware errors to clients is deprecated ' + - 'and will be removed in a future release. Disable this behaviour with:\n\n' + - 'new Backend({doNotForwardSendPresenceErrorsToClient: true})\n\n' - ); - } - this.doNotCommitNoOps = !!options.doNotCommitNoOps; - - // Map from event name to a list of middleware - this.middleware = Object.create(null); - - // The number of open agents for monitoring and testing memory leaks - this.agentsCount = 0; - this.remoteAgentsCount = 0; - - this.errorHandler = typeof options.errorHandler === 'function' ? - options.errorHandler : - // eslint-disable-next-line no-unused-vars - function(error, context) { - logger.error(error); - }; -} -module.exports = Backend; -emitter.mixin(Backend); - -Backend.prototype.MIDDLEWARE_ACTIONS = { - // An operation was successfully written to the database. - afterWrite: 'afterWrite', - // An operation is about to be applied to a snapshot before being committed to the database - apply: 'apply', - // An operation was applied to a snapshot; The operation and new snapshot are about to be written to the database. - commit: 'commit', - // A new client connected to the server. - connect: 'connect', - // An operation was loaded from the database - op: 'op', - // A query is about to be sent to the database - query: 'query', - // Snapshot(s) were received from the database and are about to be returned to a client - readSnapshots: 'readSnapshots', - // Received a message from a client - receive: 'receive', - // About to send a non-error reply to a client message. - // WARNING: This gets passed a direct reference to the reply object, so - // be cautious with it. While modifications to the reply message are possible - // by design, changing existing reply properties can cause weird bugs, since - // the rest of ShareDB would be unaware of those changes. - reply: 'reply', - // The server received presence information - receivePresence: 'receivePresence', - // About to send presence information to a client - sendPresence: 'sendPresence', - // An operation is about to be submitted to the database - submit: 'submit' -}; - -Backend.prototype.SNAPSHOT_TYPES = { - // The current snapshot is being fetched (eg through backend.fetch) - current: 'current', - // A specific snapshot is being fetched by version (eg through backend.fetchSnapshot) - byVersion: 'byVersion', - // A specific snapshot is being fetch by timestamp (eg through backend.fetchSnapshotByTimestamp) - byTimestamp: 'byTimestamp' -}; - -Backend.prototype.close = function(callback) { - var wait = 4; - var backend = this; - function finish(err) { - if (err) { - if (callback) return callback(err); - return backend.emit('error', err); - } - if (--wait) return; - if (callback) callback(); - } - this.pubsub.close(finish); - this.db.close(finish); - this.milestoneDb.close(finish); - for (var name in this.extraDbs) { - wait++; - this.extraDbs[name].close(finish); - } - finish(); -}; - -Backend.prototype.connect = function(connection, req, callback) { - var socket = new StreamSocket(); - if (connection) { - connection.bindToSocket(socket); - } else { - connection = new Connection(socket); - } - socket._open(); - var agent = this.listen(socket.stream, req); - // Store a reference to the agent on the connection for convenience. This is - // not used internal to ShareDB, but it is handy for server-side only user - // code that may cache state on the agent and read it in middleware - connection.agent = agent; - - if (typeof callback === 'function') { - connection.once('connected', function() { - callback(connection); - }); - } - - return connection; -}; - -/** A client has connected through the specified stream. Listen for messages. - * - * The optional second argument (req) is an initial request which is passed - * through to any connect() middleware. This is useful for inspecting cookies - * or an express session or whatever on the request object in your middleware. - * - * (The agent is available through all middleware) - */ -Backend.prototype.listen = function(stream, req) { - var agent = new Agent(this, stream); - this.trigger(this.MIDDLEWARE_ACTIONS.connect, agent, {stream: stream, req: req}, function(err) { - if (err) return agent.close(err); - agent._open(); - }); - return agent; -}; - -Backend.prototype.addProjection = function(name, collection, fields) { - if (this.projections[name]) { - throw new Error('Projection ' + name + ' already exists'); - } - - for (var key in fields) { - if (fields[key] !== true) { - throw new Error('Invalid field ' + key + ' - fields must be {somekey: true}. Subfields not currently supported.'); - } - } - - this.projections[name] = { - target: collection, - fields: fields - }; -}; - -/** - * Add middleware to an action or array of actions - */ -Backend.prototype.use = function(action, fn) { - if (Array.isArray(action)) { - for (var i = 0; i < action.length; i++) { - this.use(action[i], fn); - } - return this; - } - var fns = this.middleware[action] || (this.middleware[action] = []); - fns.push(fn); - return this; -}; - -/** - * Passes request through the middleware stack - * - * Middleware may modify the request object. After all middleware have been - * invoked we call `callback` with `null` and the modified request. If one of - * the middleware resturns an error the callback is called with that error. - */ -Backend.prototype.trigger = function(action, agent, request, callback) { - request.action = action; - request.agent = agent; - request.backend = this; - - var fns = this.middleware[action]; - if (!fns) return callback(); - - // Copying the triggers we'll fire so they don't get edited while we iterate. - fns = fns.slice(); - var next = function(err) { - if (err) return callback(err); - var fn = fns.shift(); - if (!fn) return callback(); - fn(request, next); - }; - next(); -}; - -// Submit an operation on the named collection/docname. op should contain a -// {op:}, {create:} or {del:} field. It should probably contain a v: field (if -// it doesn't, it defaults to the current version). -Backend.prototype.submit = function(agent, index, id, op, options, originalCallback) { - var backend = this; - var request = new SubmitRequest(this, agent, index, id, op, options); - - var callback = function(error, ops) { - backend.emit('submitRequestEnd', error, request); - originalCallback(error, ops, request); - }; - - var err = ot.checkOp(op); - if (err) return callback(err); - backend.trigger(backend.MIDDLEWARE_ACTIONS.submit, agent, request, function(err) { - if (err) return callback(err); - request.submit(function(err) { - if (err) return callback(err); - backend.trigger(backend.MIDDLEWARE_ACTIONS.afterWrite, agent, request, function(err) { - if (err) return callback(err); - backend._sanitizeOps(agent, request.projection, request.collection, id, request.ops, function(err) { - if (err) return callback(err); - backend.emit('timing', 'submit.total', Date.now() - request.start, request); - callback(err, request.ops); - }); - }); - }); - }); -}; - -Backend.prototype.sanitizeOp = function(agent, index, id, op, callback) { - var projection = this.projections[index]; - var collection = (projection) ? projection.target : index; - this._sanitizeOp(agent, projection, collection, id, op, callback); -}; - -Backend.prototype._sanitizeOp = function(agent, projection, collection, id, op, callback) { - if (projection) { - try { - projections.projectOp(projection.fields, op); - } catch (err) { - return callback(err); - } - } - this.trigger(this.MIDDLEWARE_ACTIONS.op, agent, {collection: collection, id: id, op: op}, callback); -}; -Backend.prototype._sanitizeOps = function(agent, projection, collection, id, ops, callback) { - var backend = this; - async.each(ops, function(op, eachCb) { - backend._sanitizeOp(agent, projection, collection, id, op, function(err) { - util.nextTick(eachCb, err); - }); - }, callback); -}; -Backend.prototype._sanitizeOpsBulk = function(agent, projection, collection, opsMap, callback) { - var backend = this; - async.forEachOf(opsMap, function(ops, id, eachCb) { - backend._sanitizeOps(agent, projection, collection, id, ops, eachCb); - }, callback); -}; - -Backend.prototype._sanitizeSnapshots = function(agent, projection, collection, snapshots, snapshotType, callback) { - if (projection) { - try { - projections.projectSnapshots(projection.fields, snapshots); - } catch (err) { - return callback(err); - } - } - - var request = new ReadSnapshotsRequest(collection, snapshots, snapshotType); - - this.trigger(this.MIDDLEWARE_ACTIONS.readSnapshots, agent, request, function(err) { - if (err) return callback(err); - // Handle "partial rejection" - "readSnapshots" middleware functions can use - // `request.rejectSnapshotRead(snapshot, error)` to reject the read of a specific snapshot. - if (request.hasSnapshotRejection()) { - err = request.getReadSnapshotsError(); - } - if (err) { - callback(err); - } else { - callback(); - } - }); -}; - -Backend.prototype._getSnapshotProjection = function(db, projection) { - return (db.projectsSnapshots) ? null : projection; -}; - -Backend.prototype._getSnapshotsFromMap = function(ids, snapshotMap) { - var snapshots = new Array(ids.length); - for (var i = 0; i < ids.length; i++) { - var id = ids[i]; - snapshots[i] = snapshotMap[id]; - } - return snapshots; -}; - -Backend.prototype._getSanitizedOps = function(agent, projection, collection, id, from, to, opsOptions, callback) { - var backend = this; - if (!opsOptions) opsOptions = {}; - if (agent) opsOptions.agentCustom = agent.custom; - backend.db.getOps(collection, id, from, to, opsOptions, function(err, ops) { - if (err) return callback(err); - backend._sanitizeOps(agent, projection, collection, id, ops, function(err) { - if (err) return callback(err); - callback(null, ops); - }); - }); -}; - -Backend.prototype._getSanitizedOpsBulk = function(agent, projection, collection, fromMap, toMap, opsOptions, callback) { - var backend = this; - if (!opsOptions) opsOptions = {}; - if (agent) opsOptions.agentCustom = agent.custom; - backend.db.getOpsBulk(collection, fromMap, toMap, opsOptions, function(err, opsMap) { - if (err) return callback(err); - backend._sanitizeOpsBulk(agent, projection, collection, opsMap, function(err) { - if (err) return callback(err); - callback(null, opsMap); - }); - }); -}; - -// Non inclusive - gets ops from [from, to). Ie, all relevant ops. If to is -// not defined (null or undefined) then it returns all ops. -Backend.prototype.getOps = function(agent, index, id, from, to, options, callback) { - if (typeof options === 'function') { - callback = options; - options = null; - } - var start = Date.now(); - var projection = this.projections[index]; - var collection = (projection) ? projection.target : index; - var backend = this; - var request = { - agent: agent, - index: index, - collection: collection, - id: id, - from: from, - to: to - }; - var opsOptions = options && options.opsOptions; - backend._getSanitizedOps(agent, projection, collection, id, from, to, opsOptions, function(err, ops) { - if (err) return callback(err); - backend.emit('timing', 'getOps', Date.now() - start, request); - callback(null, ops); - }); -}; - -Backend.prototype.getOpsBulk = function(agent, index, fromMap, toMap, options, callback) { - if (typeof options === 'function') { - callback = options; - options = null; - } - var start = Date.now(); - var projection = this.projections[index]; - var collection = (projection) ? projection.target : index; - var backend = this; - var request = { - agent: agent, - index: index, - collection: collection, - fromMap: fromMap, - toMap: toMap - }; - var opsOptions = options && options.opsOptions; - backend._getSanitizedOpsBulk(agent, projection, collection, fromMap, toMap, opsOptions, function(err, opsMap) { - if (err) return callback(err); - backend.emit('timing', 'getOpsBulk', Date.now() - start, request); - callback(null, opsMap); - }); -}; - -Backend.prototype.fetch = function(agent, index, id, options, callback) { - if (typeof options === 'function') { - callback = options; - options = null; - } - var start = Date.now(); - var projection = this.projections[index]; - var collection = (projection) ? projection.target : index; - var fields = projection && projection.fields; - var backend = this; - var request = { - agent: agent, - index: index, - collection: collection, - id: id - }; - var snapshotOptions = (options && options.snapshotOptions) || {}; - snapshotOptions.agentCustom = agent.custom; - backend.db.getSnapshot(collection, id, fields, snapshotOptions, function(err, snapshot) { - if (err) return callback(err); - var snapshotProjection = backend._getSnapshotProjection(backend.db, projection); - var snapshots = [snapshot]; - backend._sanitizeSnapshots( - agent, - snapshotProjection, - collection, - snapshots, - backend.SNAPSHOT_TYPES.current, - function(err) { - if (err) return callback(err); - backend.emit('timing', 'fetch', Date.now() - start, request); - callback(null, snapshot); - }); - }); -}; - -/** - * Map of document id to Snapshot or error object. - * @typedef {{ [id: string]: Snapshot | { error: Error | string } }} SnapshotMap - */ - -/** - * @param {Agent} agent - * @param {string} index - * @param {string[]} ids - * @param {*} options - * @param {(err?: Error | string, snapshotMap?: SnapshotMap) => void} callback - */ -Backend.prototype.fetchBulk = function(agent, index, ids, options, callback) { - if (typeof options === 'function') { - callback = options; - options = null; - } - var start = Date.now(); - var projection = this.projections[index]; - var collection = (projection) ? projection.target : index; - var fields = projection && projection.fields; - var backend = this; - var request = { - agent: agent, - index: index, - collection: collection, - ids: ids - }; - var snapshotOptions = (options && options.snapshotOptions) || {}; - snapshotOptions.agentCustom = agent.custom; - backend.db.getSnapshotBulk(collection, ids, fields, snapshotOptions, function(err, snapshotMap) { - if (err) return callback(err); - var snapshotProjection = backend._getSnapshotProjection(backend.db, projection); - var snapshots = backend._getSnapshotsFromMap(ids, snapshotMap); - backend._sanitizeSnapshots( - agent, - snapshotProjection, - collection, - snapshots, - backend.SNAPSHOT_TYPES.current, - function(err) { - if (err) { - if (err.code === ERROR_CODE.ERR_SNAPSHOT_READS_REJECTED) { - for (var docId in err.idToError) { - snapshotMap[docId] = {error: err.idToError[docId]}; - } - err = undefined; - } else { - snapshotMap = undefined; - } - } - backend.emit('timing', 'fetchBulk', Date.now() - start, request); - callback(err, snapshotMap); - }); - }); -}; - -// Subscribe to the document from the specified version or null version -Backend.prototype.subscribe = function(agent, index, id, version, options, callback) { - if (typeof options === 'function') { - callback = options; - options = null; - } - if (options) { - // We haven't yet implemented the ability to pass options to subscribe. This is because we need to - // add the ability to SubmitRequest.commit to optionally pass the metadata to other clients on - // PubSub. This behaviour is not needed right now, but we have added an options object to the - // subscribe() signature so that it remains consistent with getOps() and fetch(). - return callback(new ShareDBError( - ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, - 'Passing options to subscribe has not been implemented' - )); - } - var start = Date.now(); - var projection = this.projections[index]; - var collection = (projection) ? projection.target : index; - var channel = this.getDocChannel(collection, id); - var backend = this; - var request = { - agent: agent, - index: index, - collection: collection, - id: id, - version: version - }; - backend.pubsub.subscribe(channel, function(err, stream) { - if (err) return callback(err); - if (version == null) { - // Subscribing from null means that the agent doesn't have a document - // and needs to fetch it as well as subscribing - backend.fetch(agent, index, id, function(err, snapshot) { - if (err) { - stream.destroy(); - return callback(err); - } - backend.emit('timing', 'subscribe.snapshot', Date.now() - start, request); - callback(null, stream, snapshot); - }); - } else { - backend._getSanitizedOps(agent, projection, collection, id, version, null, null, function(err, ops) { - if (err) { - stream.destroy(); - return callback(err); - } - backend.emit('timing', 'subscribe.ops', Date.now() - start, request); - callback(null, stream, null, ops); - }); - } - }); -}; - -/** - * Map of document id to pubsub stream. - * @typedef {{ [id: string]: Stream }} StreamMap - */ -/** - * Map of document id to array of ops for the doc. - * @typedef {{ [id: string]: Op[] }} OpsMap - */ - -/** - * @param {Agent} agent - * @param {string} index - * @param {string[]} versions - * @param {( - * err?: Error | string | null, - * streams?: StreamMap, - * snapshotMap?: SnapshotMap | null - * opsMap?: OpsMap - * ) => void} callback - */ -Backend.prototype.subscribeBulk = function(agent, index, versions, callback) { - var start = Date.now(); - var projection = this.projections[index]; - var collection = (projection) ? projection.target : index; - var backend = this; - var streams = Object.create(null); - var doFetch = Array.isArray(versions); - var ids = (doFetch) ? versions : Object.keys(versions); - var request = { - agent: agent, - index: index, - collection: collection, - versions: versions - }; - async.each(ids, function(id, eachCb) { - var channel = backend.getDocChannel(collection, id); - backend.pubsub.subscribe(channel, function(err, stream) { - if (err) return eachCb(err); - streams[id] = stream; - eachCb(); - }); - }, function(err) { - if (err) { - destroyStreams(streams); - return callback(err); - } - if (doFetch) { - // If an array of ids, get current snapshots - backend.fetchBulk(agent, index, ids, function(err, snapshotMap) { - if (err) { - // Full error, destroy all streams. - destroyStreams(streams); - streams = undefined; - snapshotMap = undefined; - } - for (var docId in snapshotMap) { - // The doc id could map to an object `{error: Error | string}`, which indicates that - // particular snapshot's read was rejected. Destroy the streams fur such docs. - if (snapshotMap[docId].error) { - streams[docId].destroy(); - delete streams[docId]; - } - } - backend.emit('timing', 'subscribeBulk.snapshot', Date.now() - start, request); - callback(err, streams, snapshotMap); - }); - } else { - // If a versions map, get ops since requested versions - backend._getSanitizedOpsBulk(agent, projection, collection, versions, null, null, function(err, opsMap) { - if (err) { - destroyStreams(streams); - return callback(err); - } - backend.emit('timing', 'subscribeBulk.ops', Date.now() - start, request); - callback(null, streams, null, opsMap); - }); - } - }); -}; -function destroyStreams(streams) { - for (var id in streams) { - streams[id].destroy(); - } -} - -Backend.prototype.queryFetch = function(agent, index, query, options, callback) { - var start = Date.now(); - var backend = this; - backend._triggerQuery(agent, index, query, options, function(err, request) { - if (err) return callback(err); - backend._query(agent, request, function(err, snapshots, extra) { - if (err) return callback(err); - backend.emit('timing', 'queryFetch', Date.now() - start, request); - callback(null, snapshots, extra); - }); - }); -}; - -// Options can contain: -// db: The name of the DB (if the DB is specified in the otherDbs when the backend instance is created) -// skipPoll: function(collection, id, op, query) {return true or false; } -// this is a synchronous function which can be used as an early filter for -// operations going through the system to reduce the load on the DB. -// pollDebounce: Minimum delay between subsequent database polls. This is -// used to batch updates to reduce load on the database at the expense of -// liveness -Backend.prototype.querySubscribe = function(agent, index, query, options, callback) { - var start = Date.now(); - var backend = this; - backend._triggerQuery(agent, index, query, options, function(err, request) { - if (err) return callback(err); - if (request.db.disableSubscribe) { - return callback(new ShareDBError( - ERROR_CODE.ERR_DATABASE_DOES_NOT_SUPPORT_SUBSCRIBE, - 'DB does not support subscribe' - )); - } - - var channels = request.channels; - - if (request.channel) { - logger.warn( - '[DEPRECATED] "query" middleware\'s context.channel is deprecated, use context.channels instead. ' + - 'Read more: https://share.github.io/sharedb/middleware/actions#query' - ); - channels = [request.channel]; - } - - if (!channels || !channels.length) { - return callback(new ShareDBError(ERROR_CODE.ERR_QUERY_CHANNEL_MISSING, 'Required minimum one query channel.')); - } - - var streams = []; - - function destroyStreams() { - streams.forEach(function(stream) { - stream.destroy(); - }); - } - - function createQueryEmitter() { - if (options.ids) { - var queryEmitter = new QueryEmitter(request, streams, options.ids); - backend.emit('timing', 'querySubscribe.reconnect', Date.now() - start, request); - callback(null, queryEmitter); - return; - } - // Issue query on db to get our initial results - backend._query(agent, request, function(err, snapshots, extra) { - if (err) { - destroyStreams(); - return callback(err); - } - var ids = pluckIds(snapshots); - var queryEmitter = new QueryEmitter(request, streams, ids, extra); - backend.emit('timing', 'querySubscribe.initial', Date.now() - start, request); - callback(null, queryEmitter, snapshots, extra); - }); - } - - channels.forEach(function(channel) { - backend.pubsub.subscribe(channel, function(err, stream) { - if (err) { - destroyStreams(); - return callback(err); - } - streams.push(stream); - - var subscribedToAllChannels = streams.length === channels.length; - if (subscribedToAllChannels) { - createQueryEmitter(); - } - }); - }); - }); -}; - -Backend.prototype._triggerQuery = function(agent, index, query, options, callback) { - var projection = this.projections[index]; - var collection = (projection) ? projection.target : index; - var fields = projection && projection.fields; - var request = { - index: index, - collection: collection, - projection: projection, - fields: fields, - channels: [this.getCollectionChannel(collection)], - query: query, - options: options, - db: null, - snapshotProjection: null - }; - var backend = this; - backend.trigger(backend.MIDDLEWARE_ACTIONS.query, agent, request, function(err) { - if (err) return callback(err); - // Set the DB reference for the request after the middleware trigger so - // that the db option can be changed in middleware - request.db = (options.db) ? backend.extraDbs[options.db] : backend.db; - if (!request.db) return callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_ADAPTER_NOT_FOUND, 'DB not found')); - request.snapshotProjection = backend._getSnapshotProjection(request.db, projection); - callback(null, request); - }); -}; - -Backend.prototype._query = function(agent, request, callback) { - var backend = this; - request.db.query(request.collection, request.query, request.fields, request.options, function(err, snapshots, extra) { - if (err) return callback(err); - backend._sanitizeSnapshots( - agent, - request.snapshotProjection, - request.collection, - snapshots, - backend.SNAPSHOT_TYPES.current, - function(err) { - callback(err, snapshots, extra); - }); - }); -}; - -Backend.prototype.getCollectionChannel = function(collection) { - return collection; -}; - -Backend.prototype.getDocChannel = function(collection, id) { - return collection + '.' + id; -}; - -Backend.prototype.getChannels = function(collection, id) { - return [ - this.getCollectionChannel(collection), - this.getDocChannel(collection, id) - ]; -}; - -Backend.prototype.fetchSnapshot = function(agent, index, id, version, callback) { - var start = Date.now(); - var backend = this; - var projection = this.projections[index]; - var collection = projection ? projection.target : index; - var request = { - agent: agent, - index: index, - collection: collection, - id: id, - version: version - }; - - this._fetchSnapshot(collection, id, version, function(error, snapshot) { - if (error) return callback(error); - var snapshotProjection = backend._getSnapshotProjection(backend.db, projection); - var snapshots = [snapshot]; - var snapshotType = backend.SNAPSHOT_TYPES.byVersion; - backend._sanitizeSnapshots(agent, snapshotProjection, collection, snapshots, snapshotType, function(error) { - if (error) return callback(error); - backend.emit('timing', 'fetchSnapshot', Date.now() - start, request); - callback(null, snapshot); - }); - }); -}; - -Backend.prototype._fetchSnapshot = function(collection, id, version, callback) { - var db = this.db; - var backend = this; - - var shouldGetLatestSnapshot = version === null; - if (shouldGetLatestSnapshot) { - return backend.db.getSnapshot(collection, id, null, null, function(error, snapshot) { - if (error) return callback(error); - - callback(null, snapshot); - }); - } - - - this.milestoneDb.getMilestoneSnapshot(collection, id, version, function(error, milestoneSnapshot) { - if (error) return callback(error); - - // Bypass backend.getOps so that we don't call _sanitizeOps. We want to avoid this, because: - // - we want to avoid the 'op' middleware, because we later use the 'readSnapshots' middleware in _sanitizeSnapshots - // - we handle the projection in _sanitizeSnapshots - var from = milestoneSnapshot ? milestoneSnapshot.v : 0; - db.getOps(collection, id, from, version, null, function(error, ops) { - if (error) return callback(error); - - backend._buildSnapshotFromOps(id, milestoneSnapshot, ops, function(error, snapshot) { - if (error) return callback(error); - - if (version > snapshot.v) { - return callback(new ShareDBError( - ERROR_CODE.ERR_OP_VERSION_NEWER_THAN_CURRENT_SNAPSHOT, - 'Requested version exceeds latest snapshot version' - )); - } - - callback(null, snapshot); - }); - }); - }); -}; - -Backend.prototype.fetchSnapshotByTimestamp = function(agent, index, id, timestamp, callback) { - var start = Date.now(); - var backend = this; - var projection = this.projections[index]; - var collection = projection ? projection.target : index; - var request = { - agent: agent, - index: index, - collection: collection, - id: id, - timestamp: timestamp - }; - - this._fetchSnapshotByTimestamp(collection, id, timestamp, function(error, snapshot) { - if (error) return callback(error); - var snapshotProjection = backend._getSnapshotProjection(backend.db, projection); - var snapshots = [snapshot]; - var snapshotType = backend.SNAPSHOT_TYPES.byTimestamp; - backend._sanitizeSnapshots(agent, snapshotProjection, collection, snapshots, snapshotType, function(error) { - if (error) return callback(error); - backend.emit('timing', 'fetchSnapshot', Date.now() - start, request); - callback(null, snapshot); - }); - }); -}; - -Backend.prototype._fetchSnapshotByTimestamp = function(collection, id, timestamp, callback) { - var db = this.db; - var milestoneDb = this.milestoneDb; - var backend = this; - - var milestoneSnapshot; - var from = 0; - var to = null; - - var shouldGetLatestSnapshot = timestamp === null; - if (shouldGetLatestSnapshot) { - return backend.db.getSnapshot(collection, id, null, null, function(error, snapshot) { - if (error) return callback(error); - - callback(null, snapshot); - }); - } - - milestoneDb.getMilestoneSnapshotAtOrBeforeTime(collection, id, timestamp, function(error, snapshot) { - if (error) return callback(error); - milestoneSnapshot = snapshot; - if (snapshot) from = snapshot.v; - - milestoneDb.getMilestoneSnapshotAtOrAfterTime(collection, id, timestamp, function(error, snapshot) { - if (error) return callback(error); - if (snapshot) to = snapshot.v; - - var options = {metadata: true}; - db.getOps(collection, id, from, to, options, function(error, ops) { - if (error) return callback(error); - filterOpsInPlaceBeforeTimestamp(ops, timestamp); - backend._buildSnapshotFromOps(id, milestoneSnapshot, ops, callback); - }); - }); - }); -}; - -Backend.prototype._buildSnapshotFromOps = function(id, startingSnapshot, ops, callback) { - var snapshot = util.clone(startingSnapshot) || new Snapshot(id, 0, null, undefined, null); - var error = ot.applyOps(snapshot, ops, {_normalizeLegacyJson0Ops: true}); - callback(error, snapshot); -}; - -Backend.prototype.transformPresenceToLatestVersion = function(agent, presence, callback) { - if (!presence.c || !presence.d) return callback(null, presence); - this.getOps(agent, presence.c, presence.d, presence.v, null, function(error, ops) { - if (error) return callback(error); - for (var i = 0; i < ops.length; i++) { - var op = ops[i]; - var isOwnOp = op.src === presence.src; - var transformError = ot.transformPresence(presence, op, isOwnOp); - if (transformError) { - return callback(transformError); - } - } - callback(null, presence); - }); -}; - -function pluckIds(snapshots) { - var ids = []; - for (var i = 0; i < snapshots.length; i++) { - ids.push(snapshots[i].id); - } - return ids; -} - -function filterOpsInPlaceBeforeTimestamp(ops, timestamp) { - for (var i = 0; i < ops.length; i++) { - var op = ops[i]; - var opTimestamp = op.m && op.m.ts; - if (opTimestamp > timestamp) { - ops.length = i; - return; - } - } -} diff --git a/lib/client/connection.js b/lib/client/connection.js deleted file mode 100644 index da89dfef5..000000000 --- a/lib/client/connection.js +++ /dev/null @@ -1,840 +0,0 @@ -var Doc = require('./doc'); -var Query = require('./query'); -var Presence = require('./presence/presence'); -var DocPresence = require('./presence/doc-presence'); -var SnapshotVersionRequest = require('./snapshot-request/snapshot-version-request'); -var SnapshotTimestampRequest = require('./snapshot-request/snapshot-timestamp-request'); -var emitter = require('../emitter'); -var ShareDBError = require('../error'); -var ACTIONS = require('../message-actions').ACTIONS; -var types = require('../types'); -var util = require('../util'); -var logger = require('../logger'); -var DocPresenceEmitter = require('./presence/doc-presence-emitter'); -var protocol = require('../protocol'); - -var ERROR_CODE = ShareDBError.CODES; - -function connectionState(socket) { - if (socket.readyState === 0 || socket.readyState === 1) return 'connecting'; - return 'disconnected'; -} - -/** - * Handles communication with the sharejs server and provides queries and - * documents. - * - * We create a connection with a socket object - * connection = new sharejs.Connection(sockset) - * The socket may be any object handling the websocket protocol. See the - * documentation of bindToSocket() for details. We then wait for the connection - * to connect - * connection.on('connected', ...) - * and are finally able to work with shared documents - * connection.get('food', 'steak') // Doc - * - * @param socket @see bindToSocket - */ -module.exports = Connection; -function Connection(socket) { - emitter.EventEmitter.call(this); - - // Map of collection -> id -> doc object for created documents. - // (created documents MUST BE UNIQUE) - this.collections = Object.create(null); - - // Each query and snapshot request is created with an id that the server uses when it sends us - // info about the request (updates, etc) - this.nextQueryId = 1; - this.nextSnapshotRequestId = 1; - - // Map from query ID -> query object. - this.queries = Object.create(null); - - // Maps from channel -> presence objects - this._presences = Object.create(null); - this._docPresenceEmitter = new DocPresenceEmitter(); - - // Map from snapshot request ID -> snapshot request - this._snapshotRequests = Object.create(null); - - // A unique message number for the given id - this.seq = 1; - - // A unique message number for presence - this._presenceSeq = 1; - - // Equals agent.src on the server - this.id = null; - - // This direct reference from connection to agent is not used internal to - // ShareDB, but it is handy for server-side only user code that may cache - // state on the agent and read it in middleware - this.agent = null; - - this.debug = false; - - this.state = connectionState(socket); - - this.bindToSocket(socket); -} -emitter.mixin(Connection); - - -/** - * Use socket to communicate with server - * - * Socket is an object that can handle the websocket protocol. This method - * installs the onopen, onclose, onmessage and onerror handlers on the socket to - * handle communication and sends messages by calling socket.send(message). The - * sockets `readyState` property is used to determine the initaial state. - * - * @param socket Handles the websocket protocol - * @param socket.readyState - * @param socket.close - * @param socket.send - * @param socket.onopen - * @param socket.onclose - * @param socket.onmessage - * @param socket.onerror - */ -Connection.prototype.bindToSocket = function(socket) { - if (this.socket) { - this.socket.close(); - this.socket.onmessage = null; - this.socket.onopen = null; - this.socket.onerror = null; - this.socket.onclose = null; - } - - this.socket = socket; - - // State of the connection. The corresponding events are emitted when this changes - // - // - 'connecting' The connection is still being established, or we are still - // waiting on the server to send us the initialization message - // - 'connected' The connection is open and we have connected to a server - // and recieved the initialization message - // - 'disconnected' Connection is closed, but it will reconnect automatically - // - 'closed' The connection was closed by the client, and will not reconnect - // - 'stopped' The connection was closed by the server, and will not reconnect - var newState = connectionState(socket); - this._setState(newState); - - // This is a helper variable the document uses to see whether we're - // currently in a 'live' state. It is true if and only if we're connected - this.canSend = false; - - var connection = this; - - socket.onmessage = function(event) { - try { - var data = (typeof event.data === 'string') ? - JSON.parse(event.data) : event.data; - } catch (err) { - logger.warn('Failed to parse message', event); - return; - } - - if (connection.debug) logger.info('RECV', JSON.stringify(data)); - - var request = {data: data}; - connection.emit('receive', request); - if (!request.data) return; - - try { - connection.handleMessage(request.data); - } catch (err) { - util.nextTick(function() { - connection.emit('error', err); - }); - } - }; - - // If socket is already open, do handshake immediately. - if (socket.readyState === 1) { - connection._initializeHandshake(); - } - socket.onopen = function() { - connection._setState('connecting'); - connection._initializeHandshake(); - }; - - socket.onerror = function(err) { - // This isn't the same as a regular error, because it will happen normally - // from time to time. Your connection should probably automatically - // reconnect anyway, but that should be triggered off onclose not onerror. - // (onclose happens when onerror gets called anyway). - connection.emit('connection error', err); - }; - - socket.onclose = function(reason) { - // node-browserchannel reason values: - // 'Closed' - The socket was manually closed by calling socket.close() - // 'Stopped by server' - The server sent the stop message to tell the client not to try connecting - // 'Request failed' - Server didn't respond to request (temporary, usually offline) - // 'Unknown session ID' - Server session for client is missing (temporary, will immediately reestablish) - - if (reason === 'closed' || reason === 'Closed') { - connection._setState('closed', reason); - } else if (reason === 'stopped' || reason === 'Stopped by server') { - connection._setState('stopped', reason); - } else { - connection._setState('disconnected', reason); - } - }; -}; - -/** - * @param {object} message - * @param {string} message.a action - */ -Connection.prototype.handleMessage = function(message) { - var err = null; - if (message.error) { - err = wrapErrorData(message.error, message); - delete message.error; - } - // Switch on the message action. Most messages are for documents and are - // handled in the doc class. - switch (message.a) { - case ACTIONS.initLegacy: - // Client initialization packet - return this._handleLegacyInit(message); - case ACTIONS.handshake: - return this._handleHandshake(err, message); - case ACTIONS.queryFetch: - var query = this.queries[message.id]; - if (query) query._handleFetch(err, message.data, message.extra); - return; - case ACTIONS.querySubscribe: - var query = this.queries[message.id]; - if (query) query._handleSubscribe(err, message.data, message.extra); - return; - case ACTIONS.queryUnsubscribe: - // Queries are removed immediately on calls to destroy, so we ignore - // replies to query unsubscribes. Perhaps there should be a callback for - // destroy, but this is currently unimplemented - return; - case ACTIONS.queryUpdate: - // Query message. Pass this to the appropriate query object. - var query = this.queries[message.id]; - if (!query) return; - if (err) return query._handleError(err); - if (message.diff) query._handleDiff(message.diff); - if (util.hasOwn(message, 'extra')) query._handleExtra(message.extra); - return; - - case ACTIONS.bulkFetch: - return this._handleBulkMessage(err, message, '_handleFetch'); - case ACTIONS.bulkSubscribe: - case ACTIONS.bulkUnsubscribe: - return this._handleBulkMessage(err, message, '_handleSubscribe'); - - case ACTIONS.snapshotFetch: - case ACTIONS.snapshotFetchByTimestamp: - return this._handleSnapshotFetch(err, message); - - case ACTIONS.fetch: - var doc = this.getExisting(message.c, message.d); - if (doc) doc._handleFetch(err, message.data); - return; - case ACTIONS.subscribe: - case ACTIONS.unsubscribe: - var doc = this.getExisting(message.c, message.d); - if (doc) doc._handleSubscribe(err, message.data); - return; - case ACTIONS.op: - var doc = this.getExisting(message.c, message.d); - if (doc) doc._handleOp(err, message); - return; - case ACTIONS.presence: - return this._handlePresence(err, message); - case ACTIONS.presenceSubscribe: - return this._handlePresenceSubscribe(err, message); - case ACTIONS.presenceUnsubscribe: - return this._handlePresenceUnsubscribe(err, message); - case ACTIONS.presenceRequest: - return this._handlePresenceRequest(err, message); - case ACTIONS.pingPong: - return this._handlePingPong(err); - - default: - logger.warn('Ignoring unrecognized message', message); - } -}; - -function wrapErrorData(errorData, fullMessage) { - // wrap in Error object so can be passed through event emitters - var err = new Error(errorData.message); - err.code = errorData.code; - if (fullMessage) { - // Add the message data to the error object for more context - err.data = fullMessage; - } - return err; -} - -Connection.prototype._handleBulkMessage = function(err, message, method) { - if (message.data) { - for (var id in message.data) { - var dataForId = message.data[id]; - var doc = this.getExisting(message.c, id); - if (doc) { - if (err) { - doc[method](err); - } else if (dataForId.error) { - // Bulk reply snapshot-specific errorr - see agent.js getMapResult - doc[method](wrapErrorData(dataForId.error)); - } else { - doc[method](null, dataForId); - } - } - } - } else if (Array.isArray(message.b)) { - for (var i = 0; i < message.b.length; i++) { - var id = message.b[i]; - var doc = this.getExisting(message.c, id); - if (doc) doc[method](err); - } - } else if (message.b) { - for (var id in message.b) { - var doc = this.getExisting(message.c, id); - if (doc) doc[method](err); - } - } else { - logger.error('Invalid bulk message', message); - } -}; - -Connection.prototype._reset = function() { - this.agent = null; -}; - -// Set the connection's state. The connection is basically a state machine. -Connection.prototype._setState = function(newState, reason) { - if (this.state === newState) return; - - // I made a state diagram. The only invalid transitions are getting to - // 'connecting' from anywhere other than 'disconnected' and getting to - // 'connected' from anywhere other than 'connecting'. - if ( - ( - newState === 'connecting' && - this.state !== 'disconnected' && - this.state !== 'stopped' && - this.state !== 'closed' - ) || ( - newState === 'connected' && - this.state !== 'connecting' - ) - ) { - var err = new ShareDBError( - ERROR_CODE.ERR_CONNECTION_STATE_TRANSITION_INVALID, - 'Cannot transition directly from ' + this.state + ' to ' + newState - ); - return this.emit('error', err); - } - - this.state = newState; - this.canSend = (newState === 'connected'); - - if ( - newState === 'disconnected' || - newState === 'stopped' || - newState === 'closed' - ) { - this._reset(); - } - - // Group subscribes together to help server make more efficient calls - this.startBulk(); - // Emit the event to all queries - for (var id in this.queries) { - var query = this.queries[id]; - query._onConnectionStateChanged(); - } - // Emit the event to all documents - for (var collection in this.collections) { - var docs = this.collections[collection]; - for (var id in docs) { - docs[id]._onConnectionStateChanged(); - } - } - // Emit the event to all Presences - for (var channel in this._presences) { - this._presences[channel]._onConnectionStateChanged(); - } - // Emit the event to all snapshots - for (var id in this._snapshotRequests) { - var snapshotRequest = this._snapshotRequests[id]; - snapshotRequest._onConnectionStateChanged(); - } - this.endBulk(); - - this.emit(newState, reason); - this.emit('state', newState, reason); -}; - -Connection.prototype.startBulk = function() { - if (!this.bulk) this.bulk = Object.create(null); -}; - -Connection.prototype.endBulk = function() { - if (this.bulk) { - for (var collection in this.bulk) { - var actions = this.bulk[collection]; - this._sendBulk('f', collection, actions.f); - this._sendBulk('s', collection, actions.s); - this._sendBulk('u', collection, actions.u); - } - } - this.bulk = null; -}; - -Connection.prototype._sendBulk = function(action, collection, values) { - if (!values) return; - var ids = []; - var versions = Object.create(null); - var versionsCount = 0; - var versionId; - for (var id in values) { - var value = values[id]; - if (value == null) { - ids.push(id); - } else { - versions[id] = value; - versionId = id; - versionsCount++; - } - } - if (ids.length === 1) { - var id = ids[0]; - this.send({a: action, c: collection, d: id}); - } else if (ids.length) { - this.send({a: 'b' + action, c: collection, b: ids}); - } - if (versionsCount === 1) { - var version = versions[versionId]; - this.send({a: action, c: collection, d: versionId, v: version}); - } else if (versionsCount) { - this.send({a: 'b' + action, c: collection, b: versions}); - } -}; - -Connection.prototype._sendActions = function(action, doc, version) { - // Ensure the doc is registered so that it receives the reply message - this._addDoc(doc); - if (this.bulk) { - // Bulk subscribe - var actions = this.bulk[doc.collection] || (this.bulk[doc.collection] = Object.create(null)); - var versions = actions[action] || (actions[action] = Object.create(null)); - var isDuplicate = util.hasOwn(versions, doc.id); - versions[doc.id] = version; - return isDuplicate; - } else { - // Send single doc subscribe message - var message = {a: action, c: doc.collection, d: doc.id, v: version}; - this.send(message); - } -}; - -Connection.prototype.sendFetch = function(doc) { - return this._sendActions(ACTIONS.fetch, doc, doc.version); -}; - -Connection.prototype.sendSubscribe = function(doc) { - return this._sendActions(ACTIONS.subscribe, doc, doc.version); -}; - -Connection.prototype.sendUnsubscribe = function(doc) { - return this._sendActions(ACTIONS.unsubscribe, doc); -}; - -Connection.prototype.sendOp = function(doc, op) { - // Ensure the doc is registered so that it receives the reply message - this._addDoc(doc); - var message = { - a: ACTIONS.op, - c: doc.collection, - d: doc.id, - v: doc.version, - src: op.src, - seq: op.seq, - x: {} - }; - if ('op' in op) message.op = op.op; - if (op.create) message.create = op.create; - if (op.del) message.del = op.del; - if (doc.submitSource) message.x.source = op.source; - this.send(message); -}; - - -/** - * Sends a message down the socket - */ -Connection.prototype.send = function(message) { - if (this.debug) logger.info('SEND', JSON.stringify(message)); - - this.emit('send', message); - this.socket.send(JSON.stringify(message)); -}; - -Connection.prototype.ping = function() { - if (!this.canSend) { - throw new ShareDBError( - ERROR_CODE.ERR_CANNOT_PING_OFFLINE, - 'Socket must be CONNECTED to ping' - ); - } - - var message = { - a: ACTIONS.pingPong - }; - this.send(message); -}; - -/** - * Closes the socket and emits 'closed' - */ -Connection.prototype.close = function() { - this.socket.close(); -}; - -Connection.prototype.getExisting = function(collection, id) { - if (this.collections[collection]) return this.collections[collection][id]; -}; - - -/** - * Get or create a document. - * - * @param collection - * @param id - * @return {Doc} - */ -Connection.prototype.get = function(collection, id) { - var docs = this.collections[collection] || - (this.collections[collection] = Object.create(null)); - - var doc = docs[id]; - if (!doc) { - doc = docs[id] = new Doc(this, collection, id); - this.emit('doc', doc); - } - - doc._wantsDestroy = false; - return doc; -}; - - -/** - * Remove document from this.collections - * - * @private - */ -Connection.prototype._destroyDoc = function(doc) { - if (!doc._wantsDestroy) return; - util.digAndRemove(this.collections, doc.collection, doc.id); - doc.emit('destroy'); -}; - -Connection.prototype._addDoc = function(doc) { - var docs = this.collections[doc.collection]; - if (!docs) { - docs = this.collections[doc.collection] = Object.create(null); - } - if (docs[doc.id] !== doc) { - docs[doc.id] = doc; - } -}; - -// Helper for createFetchQuery and createSubscribeQuery, below. -Connection.prototype._createQuery = function(action, collection, q, options, callback) { - var id = this.nextQueryId++; - var query = new Query(action, this, id, collection, q, options, callback); - this.queries[id] = query; - query.send(); - return query; -}; - -// Internal function. Use query.destroy() to remove queries. -Connection.prototype._destroyQuery = function(query) { - delete this.queries[query.id]; -}; - -// The query options object can contain the following fields: -// -// db: Name of the db for the query. You can attach extraDbs to ShareDB and -// pick which one the query should hit using this parameter. - -// Create a fetch query. Fetch queries are only issued once, returning the -// results directly into the callback. -// -// The callback should have the signature function(error, results, extra) -// where results is a list of Doc objects. -Connection.prototype.createFetchQuery = function(collection, q, options, callback) { - return this._createQuery(ACTIONS.queryFetch, collection, q, options, callback); -}; - -// Create a subscribe query. Subscribe queries return with the initial data -// through the callback, then update themselves whenever the query result set -// changes via their own event emitter. -// -// If present, the callback should have the signature function(error, results, extra) -// where results is a list of Doc objects. -Connection.prototype.createSubscribeQuery = function(collection, q, options, callback) { - return this._createQuery(ACTIONS.querySubscribe, collection, q, options, callback); -}; - -Connection.prototype.hasPending = function() { - return !!( - this._firstDoc(hasPending) || - this._firstQuery(hasPending) || - this._firstSnapshotRequest() - ); -}; -function hasPending(object) { - return object.hasPending(); -} - -Connection.prototype.hasWritePending = function() { - return !!this._firstDoc(hasWritePending); -}; -function hasWritePending(object) { - return object.hasWritePending(); -} - -Connection.prototype.whenNothingPending = function(callback) { - var doc = this._firstDoc(hasPending); - if (doc) { - // If a document is found with a pending operation, wait for it to emit - // that nothing is pending anymore, and then recheck all documents again. - // We have to recheck all documents, just in case another mutation has - // been made in the meantime as a result of an event callback - doc.once('nothing pending', this._nothingPendingRetry(callback)); - return; - } - var query = this._firstQuery(hasPending); - if (query) { - query.once('ready', this._nothingPendingRetry(callback)); - return; - } - var snapshotRequest = this._firstSnapshotRequest(); - if (snapshotRequest) { - snapshotRequest.once('ready', this._nothingPendingRetry(callback)); - return; - } - // Call back when no pending operations - util.nextTick(callback); -}; -Connection.prototype._nothingPendingRetry = function(callback) { - var connection = this; - return function() { - util.nextTick(function() { - connection.whenNothingPending(callback); - }); - }; -}; - -Connection.prototype._firstDoc = function(fn) { - for (var collection in this.collections) { - var docs = this.collections[collection]; - for (var id in docs) { - var doc = docs[id]; - if (fn(doc)) { - return doc; - } - } - } -}; - -Connection.prototype._firstQuery = function(fn) { - for (var id in this.queries) { - var query = this.queries[id]; - if (fn(query)) { - return query; - } - } -}; - -Connection.prototype._firstSnapshotRequest = function() { - for (var id in this._snapshotRequests) { - return this._snapshotRequests[id]; - } -}; - -/** - * Fetch a read-only snapshot at a given version - * - * @param collection - the collection name of the snapshot - * @param id - the ID of the snapshot - * @param version (optional) - the version number to fetch. If null, the latest version is fetched. - * @param callback - (error, snapshot) => void, where snapshot takes the following schema: - * - * { - * id: string; // ID of the snapshot - * v: number; // version number of the snapshot - * type: string; // the OT type of the snapshot, or null if it doesn't exist or is deleted - * data: any; // the snapshot - * } - * - */ -Connection.prototype.fetchSnapshot = function(collection, id, version, callback) { - if (typeof version === 'function') { - callback = version; - version = null; - } - - var requestId = this.nextSnapshotRequestId++; - var snapshotRequest = new SnapshotVersionRequest(this, requestId, collection, id, version, callback); - this._snapshotRequests[snapshotRequest.requestId] = snapshotRequest; - snapshotRequest.send(); -}; - -/** - * Fetch a read-only snapshot at a given timestamp - * - * @param collection - the collection name of the snapshot - * @param id - the ID of the snapshot - * @param timestamp (optional) - the timestamp to fetch. If null, the latest version is fetched. - * @param callback - (error, snapshot) => void, where snapshot takes the following schema: - * - * { - * id: string; // ID of the snapshot - * v: number; // version number of the snapshot - * type: string; // the OT type of the snapshot, or null if it doesn't exist or is deleted - * data: any; // the snapshot - * } - * - */ -Connection.prototype.fetchSnapshotByTimestamp = function(collection, id, timestamp, callback) { - if (typeof timestamp === 'function') { - callback = timestamp; - timestamp = null; - } - - var requestId = this.nextSnapshotRequestId++; - var snapshotRequest = new SnapshotTimestampRequest(this, requestId, collection, id, timestamp, callback); - this._snapshotRequests[snapshotRequest.requestId] = snapshotRequest; - snapshotRequest.send(); -}; - -Connection.prototype._handleSnapshotFetch = function(error, message) { - var snapshotRequest = this._snapshotRequests[message.id]; - if (!snapshotRequest) return; - delete this._snapshotRequests[message.id]; - snapshotRequest._handleResponse(error, message); -}; - -Connection.prototype._handleLegacyInit = function(message) { - // If the protocol is at least 1.1, we want to use the - // new handshake protocol. Let's send a handshake initialize, because - // we now know the server is ready. If we've already sent it, we'll - // just ignore the response anyway. - if (protocol.checkAtLeast(message, '1.1')) return this._initializeHandshake(); - this._initialize(message); -}; - -Connection.prototype._initializeHandshake = function() { - this.send({ - a: ACTIONS.handshake, - id: this.id, - protocol: protocol.major, - protocolMinor: protocol.minor - }); -}; - -Connection.prototype._handleHandshake = function(error, message) { - if (error) return this.emit('error', error); - this._initialize(message); -}; - -Connection.prototype._handlePingPong = function(error) { - if (error) return this.emit('error', error); - this.emit('pong'); -}; - -Connection.prototype._initialize = function(message) { - if (this.state !== 'connecting') return; - - if (message.protocol !== protocol.major) { - return this.emit('error', new ShareDBError( - ERROR_CODE.ERR_PROTOCOL_VERSION_NOT_SUPPORTED, - 'Unsupported protocol version: ' + message.protocol - )); - } - if (types.map[message.type] !== types.defaultType) { - return this.emit('error', new ShareDBError( - ERROR_CODE.ERR_DEFAULT_TYPE_MISMATCH, - message.type + ' does not match the server default type' - )); - } - if (typeof message.id !== 'string') { - return this.emit('error', new ShareDBError( - ERROR_CODE.ERR_CLIENT_ID_BADLY_FORMED, - 'Client id must be a string' - )); - } - this.id = message.id; - - this._setState('connected'); -}; - -Connection.prototype.getPresence = function(channel) { - var connection = this; - var presence = util.digOrCreate(this._presences, channel, function() { - return new Presence(connection, channel); - }); - presence._wantsDestroy = false; - return presence; -}; - -Connection.prototype.getDocPresence = function(collection, id) { - var channel = DocPresence.channel(collection, id); - var connection = this; - var presence = util.digOrCreate(this._presences, channel, function() { - return new DocPresence(connection, collection, id); - }); - presence._wantsDestroy = false; - return presence; -}; - -Connection.prototype._sendPresenceAction = function(action, seq, presence) { - // Ensure the presence is registered so that it receives the reply message - this._addPresence(presence); - var message = {a: action, ch: presence.channel, seq: seq}; - this.send(message); - return message.seq; -}; - -Connection.prototype._addPresence = function(presence) { - util.digOrCreate(this._presences, presence.channel, function() { - return presence; - }); -}; - -Connection.prototype._requestRemotePresence = function(channel) { - this.send({a: ACTIONS.presenceRequest, ch: channel}); -}; - -Connection.prototype._handlePresenceSubscribe = function(error, message) { - var presence = util.dig(this._presences, message.ch); - if (presence) presence._handleSubscribe(error, message.seq); -}; - -Connection.prototype._handlePresenceUnsubscribe = function(error, message) { - var presence = util.dig(this._presences, message.ch); - if (presence) presence._handleUnsubscribe(error, message.seq); -}; - -Connection.prototype._handlePresence = function(error, message) { - var presence = util.dig(this._presences, message.ch); - if (presence) presence._receiveUpdate(error, message); -}; - -Connection.prototype._handlePresenceRequest = function(error, message) { - var presence = util.dig(this._presences, message.ch); - if (presence) presence._broadcastAllLocalPresence(error, message); -}; diff --git a/lib/client/doc.js b/lib/client/doc.js deleted file mode 100644 index dcd48ec9c..000000000 --- a/lib/client/doc.js +++ /dev/null @@ -1,1120 +0,0 @@ -var emitter = require('../emitter'); -var logger = require('../logger'); -var ShareDBError = require('../error'); -var types = require('../types'); -var util = require('../util'); -var clone = util.clone; -var deepEqual = require('fast-deep-equal'); -var ACTIONS = require('../message-actions').ACTIONS; - -var ERROR_CODE = ShareDBError.CODES; - -/** - * A Doc is a client's view on a sharejs document. - * - * It is is uniquely identified by its `id` and `collection`. Documents - * should not be created directly. Create them with connection.get() - * - * - * Subscriptions - * ------------- - * - * We can subscribe a document to stay in sync with the server. - * doc.subscribe(function(error) { - * doc.subscribed // = true - * }) - * The server now sends us all changes concerning this document and these are - * applied to our data. If the subscription was successful the initial - * data and version sent by the server are loaded into the document. - * - * To stop listening to the changes we call `doc.unsubscribe()`. - * - * If we just want to load the data but not stay up-to-date, we call - * doc.fetch(function(error) { - * doc.data // sent by server - * }) - * - * - * Events - * ------ - * - * You can use doc.on(eventName, callback) to subscribe to the following events: - * - `before op (op, source)` Fired before a partial operation is applied to the data. - * It may be used to read the old data just before applying an operation - * - `op (op, source)` Fired after every partial operation with this operation as the - * first argument - * - `create (source)` The document was created. That means its type was - * set and it has some initial data. - * - `del (data, source)` Fired after the document is deleted, that is - * the data is null. It is passed the data before deletion as an - * argument - * - `load ()` Fired when a new snapshot is ingested from a fetch, subscribe, or query - */ - -module.exports = Doc; -function Doc(connection, collection, id) { - emitter.EventEmitter.call(this); - - this.connection = connection; - - this.collection = collection; - this.id = id; - - this.version = null; - // The OT type of this document. An uncreated document has type `null` - this.type = null; - this.data = undefined; - - // Array of callbacks or nulls as placeholders - this.inflightFetch = []; - this.inflightSubscribe = null; - this.pendingFetch = []; - this.pendingSubscribe = []; - - this._isInHardRollback = false; - - // Whether we think we are subscribed on the server. Synchronously set to - // false on calls to unsubscribe and disconnect. Should never be true when - // this.wantSubscribe is false - this.subscribed = false; - // Whether to re-establish the subscription on reconnect - this.wantSubscribe = false; - - this._wantsDestroy = false; - - // The op that is currently roundtripping to the server, or null. - // - // When the connection reconnects, the inflight op is resubmitted. - // - // This has the same format as an entry in pendingOps - this.inflightOp = null; - - // All ops that are waiting for the server to acknowledge this.inflightOp - // This used to just be a single operation, but creates & deletes can't be - // composed with regular operations. - // - // This is a list of {[create:{...}], [del:true], [op:...], callbacks:[...]} - this.pendingOps = []; - - // The applyStack enables us to track any ops submitted while we are - // applying an op incrementally. This value is an array when we are - // performing an incremental apply and null otherwise. When it is an array, - // all submitted ops should be pushed onto it. The `_otApply` method will - // reset it back to null when all incremental apply loops are complete. - this.applyStack = null; - - // Disable the default behavior of composing submitted ops. This is read at - // the time of op submit, so it may be toggled on before submitting a - // specifc op and toggled off afterward - this.preventCompose = false; - - // If set to true, the source will be submitted over the connection. This - // will also have the side-effect of only composing ops whose sources are - // equal - this.submitSource = false; - - // Prevent own ops being submitted to the server. If subscribed, remote - // ops are still received. Should be toggled through the pause() and - // resume() methods to correctly flush on resume. - this.paused = false; - - // Internal counter that gets incremented every time doc.data is updated. - // Used as a cheap way to check if doc.data has changed. - this._dataStateVersion = 0; -} -emitter.mixin(Doc); - -Doc.prototype.destroy = function(callback) { - this._wantsDestroy = true; - var doc = this; - doc.whenNothingPending(function() { - if (doc.wantSubscribe) { - doc.unsubscribe(function(err) { - if (err) { - if (callback) return callback(err); - return doc.emit('error', err); - } - doc.connection._destroyDoc(doc); - if (callback) callback(); - }); - } else { - doc.connection._destroyDoc(doc); - if (callback) callback(); - } - }); -}; - - -// ****** Manipulating the document data, version and type. - -// Set the document's type, and associated properties. Most of the logic in -// this function exists to update the document based on any added & removed API -// methods. -// -// @param newType OT type provided by the ottypes library or its name or uri -Doc.prototype._setType = function(newType) { - if (typeof newType === 'string') { - newType = types.map[newType]; - } - - if (newType) { - this.type = newType; - } else if (newType === null) { - this.type = newType; - // If we removed the type from the object, also remove its data - this._setData(undefined); - } else { - var err = new ShareDBError(ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, 'Missing type ' + newType); - return this.emit('error', err); - } -}; - -Doc.prototype._setData = function(data) { - this.data = data; - this._dataStateVersion++; -}; - -// Ingest snapshot data. This data must include a version, snapshot and type. -// This is used both to ingest data that was exported with a webpage and data -// that was received from the server during a fetch. -// -// @param snapshot.v version -// @param snapshot.data -// @param snapshot.type -// @param callback -Doc.prototype.ingestSnapshot = function(snapshot, callback) { - if (!snapshot) return callback && callback(); - - if (typeof snapshot.v !== 'number') { - var err = new ShareDBError( - ERROR_CODE.ERR_INGESTED_SNAPSHOT_HAS_NO_VERSION, - 'Missing version in ingested snapshot. ' + this.collection + '.' + this.id - ); - if (callback) return callback(err); - return this.emit('error', err); - } - - // If the doc is already created or there are ops pending, we cannot use the - // ingested snapshot and need ops in order to update the document - if (this.type || this.hasWritePending()) { - // The version should only be null on a created document when it was - // created locally without fetching - if (this.version == null) { - if (this.hasWritePending()) { - // If we have pending ops and we get a snapshot for a locally created - // document, we have to wait for the pending ops to complete, because - // we don't know what version to fetch ops from. It is possible that - // the snapshot came from our local op, but it is also possible that - // the doc was created remotely (which would conflict and be an error) - return callback && this.once('no write pending', callback); - } - // Otherwise, we've encounted an error state - var err = new ShareDBError( - ERROR_CODE.ERR_DOC_MISSING_VERSION, - 'Cannot ingest snapshot in doc with null version. ' + this.collection + '.' + this.id - ); - if (callback) return callback(err); - return this.emit('error', err); - } - // If we got a snapshot for a version further along than the document is - // currently, issue a fetch to get the latest ops and catch us up - if (snapshot.v > this.version) return this.fetch(callback); - return callback && callback(); - } - - // Ignore the snapshot if we are already at a newer version. Under no - // circumstance should we ever set the current version backward - if (this.version > snapshot.v) return callback && callback(); - - this.version = snapshot.v; - var type = (snapshot.type === undefined) ? types.defaultType : snapshot.type; - this._setType(type); - this._setData( - (this.type && this.type.deserialize) ? - this.type.deserialize(snapshot.data) : - snapshot.data - ); - this.emit('load'); - callback && callback(); -}; - -Doc.prototype.whenNothingPending = function(callback) { - var doc = this; - util.nextTick(function() { - if (doc.hasPending()) { - doc.once('nothing pending', callback); - return; - } - callback(); - }); -}; - -Doc.prototype.hasPending = function() { - return !!( - this.inflightOp || - this.pendingOps.length || - this.inflightFetch.length || - this.inflightSubscribe || - this.pendingFetch.length || - this.pendingSubscribe.length - ); -}; - -Doc.prototype.hasWritePending = function() { - return !!(this.inflightOp || this.pendingOps.length); -}; - -Doc.prototype._emitNothingPending = function() { - if (this.hasWritePending()) return; - this.emit('no write pending'); - if (this.hasPending()) return; - this.emit('nothing pending'); -}; - -// **** Helpers for network messages - -Doc.prototype._emitResponseError = function(err, callback) { - if (err && err.code === ERROR_CODE.ERR_SNAPSHOT_READ_SILENT_REJECTION) { - this.wantSubscribe = false; - if (callback) { - callback(); - } - this._emitNothingPending(); - return; - } - if (callback) { - callback(err); - this._emitNothingPending(); - return; - } - this._emitNothingPending(); - this.emit('error', err); -}; - -Doc.prototype._handleFetch = function(error, snapshot) { - var callbacks = this.pendingFetch; - this.pendingFetch = []; - var callback = this.inflightFetch.shift(); - if (callback) callbacks.unshift(callback); - if (callbacks.length) { - callback = function(error) { - util.callEach(callbacks, error); - }; - } - if (error) return this._emitResponseError(error, callback); - this.ingestSnapshot(snapshot, callback); - this._emitNothingPending(); -}; - -Doc.prototype._handleSubscribe = function(error, snapshot) { - var request = this.inflightSubscribe; - this.inflightSubscribe = null; - var callbacks = this.pendingFetch; - this.pendingFetch = []; - if (request.callback) callbacks.push(request.callback); - var callback; - if (callbacks.length) { - callback = function(error) { - util.callEach(callbacks, error); - }; - } - if (error) return this._emitResponseError(error, callback); - this.subscribed = request.wantSubscribe; - if (this.subscribed) this.ingestSnapshot(snapshot, callback); - else if (callback) callback(); - this._emitNothingPending(); - this._flushSubscribe(); -}; - -Doc.prototype._handleOp = function(err, message) { - if (err) { - if (err.code === ERROR_CODE.ERR_NO_OP && message.seq === this.inflightOp.seq) { - // Our op was a no-op, either because we submitted a no-op, or - more - // likely - because our op was transformed into a no-op by the server - // because of a similar remote op. In this case, the server has avoided - // committing the op to the database, and we should just clear the in-flight - // op and call the callbacks. However, let's first catch ourselves up to - // the remote, so that we're in a nice consistent state - return this.fetch(this._clearInflightOp.bind(this)); - } - if (this.inflightOp) { - return this._rollback(err); - } - return this.emit('error', err); - } - - if (this.inflightOp && - message.src === this.inflightOp.src && - message.seq === this.inflightOp.seq) { - // The op has already been applied locally. Just update the version - // and pending state appropriately - this._opAcknowledged(message); - return; - } - - if (this.version == null || message.v > this.version) { - // This will happen in normal operation if we become subscribed to a - // new document via a query. It can also happen if we get an op for - // a future version beyond the version we are expecting next. This - // could happen if the server doesn't publish an op for whatever reason - // or because of a race condition. In any case, we can send a fetch - // command to catch back up. - // - // Fetch only sends a new fetch command if no fetches are inflight, which - // will act as a natural debouncing so we don't send multiple fetch - // requests for many ops received at once. - this.fetch(); - return; - } - - if (message.v < this.version) { - // We can safely ignore the old (duplicate) operation. - return; - } - - if (this.inflightOp) { - var transformErr = transformX(this.inflightOp, message); - if (transformErr) return this._hardRollback(transformErr); - } - - for (var i = 0; i < this.pendingOps.length; i++) { - var transformErr = transformX(this.pendingOps[i], message); - if (transformErr) return this._hardRollback(transformErr); - } - - this.version++; - try { - this._otApply(message, false); - } catch (error) { - return this._hardRollback(error); - } -}; - -// Called whenever (you guessed it!) the connection state changes. This will -// happen when we get disconnected & reconnect. -Doc.prototype._onConnectionStateChanged = function() { - if (this.connection.canSend) { - this.flush(); - this._resubscribe(); - } else { - if (this.inflightOp) { - this.pendingOps.unshift(this.inflightOp); - this.inflightOp = null; - } - this.subscribed = false; - if (this.inflightSubscribe) { - if (this.inflightSubscribe.wantSubscribe) { - this.pendingSubscribe.unshift(this.inflightSubscribe); - this.inflightSubscribe = null; - } else { - this._handleSubscribe(); - } - } - if (this.inflightFetch.length) { - this.pendingFetch = this.pendingFetch.concat(this.inflightFetch); - this.inflightFetch.length = 0; - } - } -}; - -Doc.prototype._resubscribe = function() { - if (!this.pendingSubscribe.length && this.wantSubscribe) { - return this.subscribe(); - } - var willFetch = this.pendingSubscribe.some(function(request) { - return request.wantSubscribe; - }); - if (!willFetch && this.pendingFetch.length) this.fetch(); - this._flushSubscribe(); -}; - -// Request the current document snapshot or ops that bring us up to date -Doc.prototype.fetch = function(callback) { - this._fetch({}, callback); -}; - -Doc.prototype._fetch = function(options, callback) { - this.pendingFetch.push(callback); - var shouldSend = this.connection.canSend && ( - options.force || !this.inflightFetch.length - ); - if (!shouldSend) return; - this.inflightFetch.push(this.pendingFetch.shift()); - this.connection.sendFetch(this); -}; - -// Fetch the initial document and keep receiving updates -Doc.prototype.subscribe = function(callback) { - var wantSubscribe = true; - this._queueSubscribe(wantSubscribe, callback); -}; - -// Unsubscribe. The data will stay around in local memory, but we'll stop -// receiving updates -Doc.prototype.unsubscribe = function(callback) { - var wantSubscribe = false; - this._queueSubscribe(wantSubscribe, callback); -}; - -Doc.prototype._queueSubscribe = function(wantSubscribe, callback) { - var lastRequest = this.pendingSubscribe[this.pendingSubscribe.length - 1] || this.inflightSubscribe; - var isDuplicateRequest = lastRequest && lastRequest.wantSubscribe === wantSubscribe; - if (isDuplicateRequest) { - lastRequest.callback = combineCallbacks([lastRequest.callback, callback]); - return; - } - this.pendingSubscribe.push({ - wantSubscribe: !!wantSubscribe, - callback: callback - }); - this._flushSubscribe(); -}; - -Doc.prototype._flushSubscribe = function() { - if (this.inflightSubscribe || !this.pendingSubscribe.length) return; - - if (this.connection.canSend) { - this.inflightSubscribe = this.pendingSubscribe.shift(); - this.wantSubscribe = this.inflightSubscribe.wantSubscribe; - if (this.wantSubscribe) { - this.connection.sendSubscribe(this); - } else { - // Be conservative about our subscription state. We'll be unsubscribed - // some time between sending this request, and receiving the callback, - // so let's just set ourselves to unsubscribed now. - this.subscribed = false; - this.connection.sendUnsubscribe(this); - } - - return; - } - - // If we're offline, then we're already unsubscribed. Therefore, call back - // the next request immediately if it's an unsubscribe request. - if (!this.pendingSubscribe[0].wantSubscribe) { - this.inflightSubscribe = this.pendingSubscribe.shift(); - var doc = this; - util.nextTick(function() { - doc._handleSubscribe(); - }); - } -}; - -function combineCallbacks(callbacks) { - callbacks = callbacks.filter(util.truthy); - if (!callbacks.length) return null; - return function(error) { - util.callEach(callbacks, error); - }; -} - - -// Operations // - -// Send the next pending op to the server, if we can. -// -// Only one operation can be in-flight at a time. If an operation is already on -// its way, or we're not currently connected, this method does nothing. -Doc.prototype.flush = function() { - // Ignore if we can't send or we are already sending an op - if (!this.connection.canSend || this.inflightOp) return; - - // Send first pending op unless paused - if (!this.paused && this.pendingOps.length) { - this._sendOp(); - } -}; - -// Helper function to set op to contain a no-op. -function setNoOp(op) { - delete op.op; - delete op.create; - delete op.del; -} - -// Transform server op data by a client op, and vice versa. Ops are edited in place. -function transformX(client, server) { - // Order of statements in this function matters. Be especially careful if - // refactoring this function - - // A client delete op should dominate if both the server and the client - // delete the document. Thus, any ops following the client delete (such as a - // subsequent create) will be maintained, since the server op is transformed - // to a no-op - if (client.del) return setNoOp(server); - - if (server.del) { - return new ShareDBError(ERROR_CODE.ERR_DOC_WAS_DELETED, 'Document was deleted'); - } - if (server.create) { - return new ShareDBError(ERROR_CODE.ERR_DOC_ALREADY_CREATED, 'Document already created'); - } - - // Ignore no-op coming from server - if (!('op' in server)) return; - - // I believe that this should not occur, but check just in case - if (client.create) { - return new ShareDBError(ERROR_CODE.ERR_DOC_ALREADY_CREATED, 'Document already created'); - } - - // They both edited the document. This is the normal case for this function - - // as in, most of the time we'll end up down here. - // - // You should be wondering why I'm using client.type instead of this.type. - // The reason is, if we get ops at an old version of the document, this.type - // might be undefined or a totally different type. By pinning the type to the - // op data, we make sure the right type has its transform function called. - if (client.type.transformX) { - var result = client.type.transformX(client.op, server.op); - client.op = result[0]; - server.op = result[1]; - } else { - var clientOp = client.type.transform(client.op, server.op, 'left'); - var serverOp = client.type.transform(server.op, client.op, 'right'); - client.op = clientOp; - server.op = serverOp; - } -}; - -/** - * Applies the operation to the snapshot - * - * If the operation is create or delete it emits `create` or `del`. Then the - * operation is applied to the snapshot and `op` and `after op` are emitted. - * If the type supports incremental updates and `this.incremental` is true we - * fire `op` after every small operation. - * - * This is the only function to fire the above mentioned events. - * - * @private - */ -Doc.prototype._otApply = function(op, source) { - if ('op' in op) { - if (!this.type) { - // Throw here, because all usage of _otApply should be wrapped with a try/catch - throw new ShareDBError( - ERROR_CODE.ERR_DOC_DOES_NOT_EXIST, - 'Cannot apply op to uncreated document. ' + this.collection + '.' + this.id - ); - } - - // NB: If we need to add another argument to this event, we should consider - // the fact that the 'op' event has op.src as its 3rd argument - this.emit('before op batch', op.op, source); - - // Iteratively apply multi-component remote operations and rollback ops - // (source === false) for the default JSON0 OT type. It could use - // type.shatter(), but since this code is so specific to use cases for the - // JSON0 type and ShareDB explicitly bundles the default type, we might as - // well write it this way and save needing to iterate through the op - // components twice. - // - // Ideally, we would not need this extra complexity. However, it is - // helpful for implementing bindings that update DOM nodes and other - // stateful objects by translating op events directly into corresponding - // mutations. Such bindings are most easily written as responding to - // individual op components one at a time in order, and it is important - // that the snapshot only include updates from the particular op component - // at the time of emission. Eliminating this would require rethinking how - // such external bindings are implemented. - if (!source && this.type === types.defaultType && op.op.length > 1) { - if (!this.applyStack) this.applyStack = []; - var stackLength = this.applyStack.length; - for (var i = 0; i < op.op.length; i++) { - var component = op.op[i]; - var componentOp = {op: [component]}; - // Apply the individual op component - this.emit('before op', componentOp.op, source, op.src); - // Transform componentOp against any ops that have been submitted - // sychronously inside of an op event handler since we began apply of - // our operation - for (var j = stackLength; j < this.applyStack.length; j++) { - var transformErr = transformX(this.applyStack[j], componentOp); - if (transformErr) return this._hardRollback(transformErr); - } - this._setData(this.type.apply(this.data, componentOp.op)); - this.emit('op', componentOp.op, source, op.src); - } - this.emit('op batch', op.op, source); - // Pop whatever was submitted since we started applying this op - this._popApplyStack(stackLength); - return; - } - - // The 'before op' event enables clients to pull any necessary data out of - // the snapshot before it gets changed - this.emit('before op', op.op, source, op.src); - // Apply the operation to the local data, mutating it in place - this._setData(this.type.apply(this.data, op.op)); - // Emit an 'op' event once the local data includes the changes from the - // op. For locally submitted ops, this will be synchronously with - // submission and before the server or other clients have received the op. - // For ops from other clients, this will be after the op has been - // committed to the database and published - this.emit('op', op.op, source, op.src); - this.emit('op batch', op.op, source); - return; - } - - if (op.create) { - this._setType(op.create.type); - if (this.type.deserialize) { - if (this.type.createDeserialized) { - this._setData(this.type.createDeserialized(op.create.data)); - } else { - this._setData(this.type.deserialize(this.type.create(op.create.data))); - } - } else { - this._setData(this.type.create(op.create.data)); - } - this.emit('create', source); - return; - } - - if (op.del) { - var oldData = this.data; - this._setType(null); - this.emit('del', oldData, source); - return; - } -}; - - -// ***** Sending operations - -// Actually send op to the server. -Doc.prototype._sendOp = function() { - if (!this.connection.canSend) return; - var src = this.connection.id; - - // When there is no inflightOp, send the first item in pendingOps. If - // there is inflightOp, try sending it again - if (!this.inflightOp) { - // Send first pending op - this.inflightOp = this.pendingOps.shift(); - } - var op = this.inflightOp; - if (!op) { - var err = new ShareDBError(ERROR_CODE.ERR_INFLIGHT_OP_MISSING, 'No op to send on call to _sendOp'); - return this.emit('error', err); - } - - // Track data for retrying ops - op.sentAt = Date.now(); - op.retries = (op.retries == null) ? 0 : op.retries + 1; - - // The src + seq number is a unique ID representing this operation. This tuple - // is used on the server to detect when ops have been sent multiple times and - // on the client to match acknowledgement of an op back to the inflightOp. - // Note that the src could be different from this.connection.id after a - // reconnect, since an op may still be pending after the reconnection and - // this.connection.id will change. In case an op is sent multiple times, we - // also need to be careful not to override the original seq value. - if (op.seq == null) { - if (this.connection.seq >= util.MAX_SAFE_INTEGER) { - return this.emit('error', new ShareDBError( - ERROR_CODE.ERR_CONNECTION_SEQ_INTEGER_OVERFLOW, - 'Connection seq has exceeded the max safe integer, maybe from being open for too long' - )); - } - - op.seq = this.connection.seq++; - } - - this.connection.sendOp(this, op); - - // src isn't needed on the first try, since the server session will have the - // same id, but it must be set on the inflightOp in case it is sent again - // after a reconnect and the connection's id has changed by then - if (op.src == null) op.src = src; -}; - - -// Queues the operation for submission to the server and applies it locally. -// -// Internal method called to do the actual work for submit(), create() and del(). -// @private -// -// @param op -// @param [op.op] -// @param [op.del] -// @param [op.create] -// @param [callback] called when operation is submitted -Doc.prototype._submit = function(op, source, callback) { - // Locally submitted ops must always have a truthy source - if (!source) source = true; - - // The op contains either op, create, delete, or none of the above (a no-op). - if ('op' in op) { - if (!this.type) { - if (this._isInHardRollback) { - var err = new ShareDBError( - ERROR_CODE.ERR_DOC_IN_HARD_ROLLBACK, - 'Cannot submit op. Document is performing hard rollback. ' + this.collection + '.' + this.id - ); - } else { - var err = new ShareDBError( - ERROR_CODE.ERR_DOC_DOES_NOT_EXIST, - 'Cannot submit op. Document has not been created. ' + this.collection + '.' + this.id - ); - } - - if (callback) return callback(err); - return this.emit('error', err); - } - // Try to normalize the op. This removes trailing skip:0's and things like that. - if (this.type.normalize) op.op = this.type.normalize(op.op); - } - - try { - this._pushOp(op, source, callback); - this._otApply(op, source); - } catch (error) { - return this._hardRollback(error); - } - - // The call to flush is delayed so if submit() is called multiple times - // synchronously, all the ops are combined before being sent to the server. - var doc = this; - util.nextTick(function() { - doc.flush(); - }); -}; - -Doc.prototype._pushOp = function(op, source, callback) { - op.source = source; - if (this.applyStack) { - // If we are in the process of incrementally applying an operation, don't - // compose the op and push it onto the applyStack so it can be transformed - // against other components from the op or ops being applied - this.applyStack.push(op); - } else { - // If the type supports composes, try to compose the operation onto the - // end of the last pending operation. - var composed = this._tryCompose(op); - if (composed) { - composed.callbacks.push(callback); - return; - } - } - // Push on to the pendingOps queue of ops to submit if we didn't compose - op.type = this.type; - op.callbacks = [callback]; - this.pendingOps.push(op); -}; - -Doc.prototype._popApplyStack = function(to) { - if (to > 0) { - this.applyStack.length = to; - return; - } - // Once we have completed the outermost apply loop, reset to null and no - // longer add ops to the applyStack as they are submitted - var op = this.applyStack[0]; - this.applyStack = null; - if (!op) return; - // Compose the ops added since the beginning of the apply stack, since we - // had to skip compose when they were originally pushed - var i = this.pendingOps.indexOf(op); - if (i === -1) return; - var ops = this.pendingOps.splice(i); - for (var i = 0; i < ops.length; i++) { - var op = ops[i]; - var composed = this._tryCompose(op); - if (composed) { - composed.callbacks = composed.callbacks.concat(op.callbacks); - } else { - this.pendingOps.push(op); - } - } -}; - -// Try to compose a submitted op into the last pending op. Returns the -// composed op if it succeeds, undefined otherwise -Doc.prototype._tryCompose = function(op) { - if (this.preventCompose) return; - - // We can only compose into the last pending op. Inflight ops have already - // been sent to the server, so we can't modify them - var last = this.pendingOps[this.pendingOps.length - 1]; - if (!last || last.sentAt) return; - - // If we're submitting the op source, we can only combine ops that have - // a matching source - if (this.submitSource && !deepEqual(op.source, last.source)) return; - - // Compose an op into a create by applying it. This effectively makes the op - // invisible, as if the document were created including the op originally - if (last.create && 'op' in op) { - last.create.data = this.type.apply(last.create.data, op.op); - return last; - } - - // Compose two ops into a single op if supported by the type. Types that - // support compose must be able to compose any two ops together - if ('op' in last && 'op' in op && this.type.compose) { - last.op = this.type.compose(last.op, op.op); - return last; - } -}; - -// *** Client OT entrypoints. - -// Submit an operation to the document. -// -// @param operation handled by the OT type -// @param options {source: ...} -// @param [callback] called after operation submitted -// -// @fires before op, op, after op -Doc.prototype.submitOp = function(component, options, callback) { - if (typeof options === 'function') { - callback = options; - options = null; - } - var op = {op: component}; - var source = options && options.source; - this._submit(op, source, callback); -}; - -// Create the document, which in ShareJS semantics means to set its type. Every -// object implicitly exists in the database but has no data and no type. Create -// sets the type of the object and can optionally set some initial data on the -// object, depending on the type. -// -// @param data initial -// @param type OT type -// @param options {source: ...} -// @param callback called when operation submitted -Doc.prototype.create = function(data, type, options, callback) { - if (typeof type === 'function') { - callback = type; - options = null; - type = null; - } else if (typeof options === 'function') { - callback = options; - options = null; - } - if (!type) { - type = types.defaultType.uri; - } - if (this.type) { - var err = new ShareDBError(ERROR_CODE.ERR_DOC_ALREADY_CREATED, 'Document already exists'); - if (callback) return callback(err); - return this.emit('error', err); - } - var op = {create: {type: type, data: data}}; - var source = options && options.source; - this._submit(op, source, callback); -}; - -// Delete the document. This creates and submits a delete operation to the -// server. Deleting resets the object's type to null and deletes its data. The -// document still exists, and still has the version it used to have before you -// deleted it (well, old version +1). -// -// @param options {source: ...} -// @param callback called when operation submitted -Doc.prototype.del = function(options, callback) { - if (typeof options === 'function') { - callback = options; - options = null; - } - if (!this.type) { - var err = new ShareDBError(ERROR_CODE.ERR_DOC_DOES_NOT_EXIST, 'Document does not exist'); - if (callback) return callback(err); - return this.emit('error', err); - } - var op = {del: true}; - var source = options && options.source; - this._submit(op, source, callback); -}; - - -// Stops the document from sending any operations to the server. -Doc.prototype.pause = function() { - this.paused = true; -}; - -// Continue sending operations to the server -Doc.prototype.resume = function() { - this.paused = false; - this.flush(); -}; - -// Create a snapshot that can be serialized, deserialized, and passed into `Doc.ingestSnapshot`. -Doc.prototype.toSnapshot = function() { - return { - v: this.version, - data: clone(this.data), - type: this.type.uri - }; -}; - -// *** Receiving operations - -// This is called when the server acknowledges an operation from the client. -Doc.prototype._opAcknowledged = function(message) { - if (this.inflightOp.create) { - this.version = message.v; - } else if (message.v !== this.version) { - // We should already be at the same version, because the server should - // have sent all the ops that have happened before acknowledging our op - logger.warn('Invalid version from server. Expected: ' + this.version + ' Received: ' + message.v, message); - - // Fetching should get us back to a working document state - return this.fetch(); - } - - if (message[ACTIONS.fixup]) { - for (var i = 0; i < message[ACTIONS.fixup].length; i++) { - var fixupOp = message[ACTIONS.fixup][i]; - - for (var j = 0; j < this.pendingOps.length; j++) { - var transformErr = transformX(this.pendingOps[i], fixupOp); - if (transformErr) return this._hardRollback(transformErr); - } - - try { - this._otApply(fixupOp, false); - } catch (error) { - return this._hardRollback(error); - } - } - } - - // The op was committed successfully. Increment the version number - this.version++; - - this._clearInflightOp(); -}; - -Doc.prototype._rollback = function(err) { - // The server has rejected submission of the current operation. Invert by - // just the inflight op if possible. If not possible to invert, cancel all - // pending ops and fetch the latest from the server to get us back into a - // working state, then call back - var op = this.inflightOp; - - if (!('op' in op && op.type.invert)) { - return this._hardRollback(err); - } - - try { - op.op = op.type.invert(op.op); - } catch (error) { - // If the op doesn't support `.invert()`, we just reload the doc - // instead of trying to locally revert it. - return this._hardRollback(err); - } - - // Transform the undo operation by any pending ops. - for (var i = 0; i < this.pendingOps.length; i++) { - var transformErr = transformX(this.pendingOps[i], op); - if (transformErr) return this._hardRollback(transformErr); - } - - // ... and apply it locally, reverting the changes. - // - // This operation is applied to look like it comes from a remote source. - // I'm still not 100% sure about this functionality, because its really a - // local op. Basically, the problem is that if the client's op is rejected - // by the server, the editor window should update to reflect the undo. - try { - this._otApply(op, false); - } catch (error) { - return this._hardRollback(error); - } - - // The server has rejected submission of the current operation. If we get - // an "Op submit rejected" error, this was done intentionally - // and we should roll back but not return an error to the user. - if (err.code === ERROR_CODE.ERR_OP_SUBMIT_REJECTED) { - return this._clearInflightOp(null); - } - - this._clearInflightOp(err); -}; - -Doc.prototype._hardRollback = function(err) { - this._isInHardRollback = true; - // Store pending ops so that we can notify their callbacks of the error. - // We combine the inflight op and the pending ops, because it's possible - // to hit a condition where we have no inflight op, but we do have pending - // ops. This can happen when an invalid op is submitted, which causes us - // to hard rollback before the pending op was flushed. - var pendingOps = this.pendingOps; - var inflightOp = this.inflightOp; - - // Cancel all pending ops and reset if we can't invert - this._setType(null); - this.version = null; - this.inflightOp = null; - this.pendingOps = []; - - // Fetch the latest version from the server to get us back into a working state - var doc = this; - this._fetch({force: true}, function(fetchError) { - doc._isInHardRollback = false; - - // We want to check that no errors are swallowed, so we check that: - // - there are callbacks to call, and - // - that every single pending op called a callback - // If there are no ops queued, or one of them didn't handle the error, - // then we emit the error. - - if (fetchError) { - // This is critical error as it means that our doc is not in usable state - // anymore, we should throw doc error. - logger.error('Hard rollback doc fetch failed.', fetchError, inflightOp); - - doc.emit('error', new ShareDBError( - ERROR_CODE.ERR_HARD_ROLLBACK_FETCH_FAILED, - 'Hard rollback fetch failed: ' + fetchError.message - )); - } - - if (err.code === ERROR_CODE.ERR_OP_SUBMIT_REJECTED) { - /** - * Handle special case of ERR_OP_SUBMIT_REJECTED - * This ensures that we resolve the main op callback and reject - * all the pending ops. This is hard rollback so all the pending ops will be - * discarded. This will ensure that the user is at least informed about it. - * more info: https://github.com/share/sharedb/pull/626 - */ - if (inflightOp) { - util.callEach(inflightOp.callbacks); - inflightOp = null; - } - - if (!pendingOps.length) return; - err = new ShareDBError( - ERROR_CODE.ERR_PENDING_OP_REMOVED_BY_OP_SUBMIT_REJECTED, - 'Discarding pending op because of hard rollback during ERR_OP_SUBMIT_REJECTED' - ); - } - - if (inflightOp) pendingOps.unshift(inflightOp); - var allOpsHadCallbacks = !!pendingOps.length; - for (var i = 0; i < pendingOps.length; i++) { - allOpsHadCallbacks = util.callEach(pendingOps[i].callbacks, err) && allOpsHadCallbacks; - } - if (err && !allOpsHadCallbacks) doc.emit('error', err); - }); -}; - -Doc.prototype._clearInflightOp = function(err) { - var inflightOp = this.inflightOp; - - this.inflightOp = null; - - var called = util.callEach(inflightOp.callbacks, err); - - this.flush(); - this._emitNothingPending(); - - if (err && !called) return this.emit('error', err); -}; - - diff --git a/lib/client/index.js b/lib/client/index.js deleted file mode 100644 index 78914acaa..000000000 --- a/lib/client/index.js +++ /dev/null @@ -1,6 +0,0 @@ -exports.Connection = require('./connection'); -exports.Doc = require('./doc'); -exports.Error = require('../error'); -exports.Query = require('./query'); -exports.types = require('../types'); -exports.logger = require('../logger'); diff --git a/lib/client/presence/doc-presence-emitter.js b/lib/client/presence/doc-presence-emitter.js deleted file mode 100644 index 2a8797cea..000000000 --- a/lib/client/presence/doc-presence-emitter.js +++ /dev/null @@ -1,75 +0,0 @@ -var util = require('../../util'); -var EventEmitter = require('events').EventEmitter; - -var EVENTS = [ - 'create', - 'del', - 'destroy', - 'load', - 'op' -]; - -module.exports = DocPresenceEmitter; - -function DocPresenceEmitter() { - this._docs = Object.create(null); - this._forwarders = Object.create(null); - this._emitters = Object.create(null); -} - -DocPresenceEmitter.prototype.addEventListener = function(doc, event, listener) { - this._registerDoc(doc); - var emitter = util.dig(this._emitters, doc.collection, doc.id); - emitter.on(event, listener); -}; - -DocPresenceEmitter.prototype.removeEventListener = function(doc, event, listener) { - var emitter = util.dig(this._emitters, doc.collection, doc.id); - if (!emitter) return; - emitter.off(event, listener); - // We'll always have at least one, because of the destroy listener - if (emitter._eventsCount === 1) this._unregisterDoc(doc); -}; - -DocPresenceEmitter.prototype._registerDoc = function(doc) { - var alreadyRegistered = true; - util.digOrCreate(this._docs, doc.collection, doc.id, function() { - alreadyRegistered = false; - return doc; - }); - - if (alreadyRegistered) return; - - var emitter = util.digOrCreate(this._emitters, doc.collection, doc.id, function() { - var e = new EventEmitter(); - // Set a high limit to avoid unnecessary warnings, but still - // retain some degree of memory leak detection - e.setMaxListeners(1000); - return e; - }); - - var self = this; - EVENTS.forEach(function(event) { - var forwarder = util.digOrCreate(self._forwarders, doc.collection, doc.id, event, function() { - return emitter.emit.bind(emitter, event); - }); - - doc.on(event, forwarder); - }); - - this.addEventListener(doc, 'destroy', this._unregisterDoc.bind(this, doc)); -}; - -DocPresenceEmitter.prototype._unregisterDoc = function(doc) { - var forwarders = util.dig(this._forwarders, doc.collection, doc.id); - for (var event in forwarders) { - doc.off(event, forwarders[event]); - } - - var emitter = util.dig(this._emitters, doc.collection, doc.id); - emitter.removeAllListeners(); - - util.digAndRemove(this._forwarders, doc.collection, doc.id); - util.digAndRemove(this._emitters, doc.collection, doc.id); - util.digAndRemove(this._docs, doc.collection, doc.id); -}; diff --git a/lib/client/presence/doc-presence.js b/lib/client/presence/doc-presence.js deleted file mode 100644 index 612893c59..000000000 --- a/lib/client/presence/doc-presence.js +++ /dev/null @@ -1,26 +0,0 @@ -var Presence = require('./presence'); -var LocalDocPresence = require('./local-doc-presence'); -var RemoteDocPresence = require('./remote-doc-presence'); - -function DocPresence(connection, collection, id) { - var channel = DocPresence.channel(collection, id); - Presence.call(this, connection, channel); - - this.collection = collection; - this.id = id; -} -module.exports = DocPresence; - -DocPresence.prototype = Object.create(Presence.prototype); - -DocPresence.channel = function(collection, id) { - return collection + '.' + id; -}; - -DocPresence.prototype._createLocalPresence = function(id) { - return new LocalDocPresence(this, id); -}; - -DocPresence.prototype._createRemotePresence = function(id) { - return new RemoteDocPresence(this, id); -}; diff --git a/lib/client/presence/local-doc-presence.js b/lib/client/presence/local-doc-presence.js deleted file mode 100644 index 1d0310be7..000000000 --- a/lib/client/presence/local-doc-presence.js +++ /dev/null @@ -1,152 +0,0 @@ -var LocalPresence = require('./local-presence'); -var ShareDBError = require('../../error'); -var util = require('../../util'); -var ERROR_CODE = ShareDBError.CODES; - -module.exports = LocalDocPresence; -function LocalDocPresence(presence, presenceId) { - LocalPresence.call(this, presence, presenceId); - - this.collection = this.presence.collection; - this.id = this.presence.id; - - this._doc = this.connection.get(this.collection, this.id); - this._emitter = this.connection._docPresenceEmitter; - this._isSending = false; - this._docDataVersionByPresenceVersion = Object.create(null); - - this._opHandler = this._transformAgainstOp.bind(this); - this._createOrDelHandler = this._handleCreateOrDel.bind(this); - this._loadHandler = this._handleLoad.bind(this); - this._destroyHandler = this.destroy.bind(this); - this._registerWithDoc(); -} - -LocalDocPresence.prototype = Object.create(LocalPresence.prototype); - -LocalDocPresence.prototype.submit = function(value, callback) { - if (!this._doc.type) { - // If the Doc hasn't been created, we already assume all presence to - // be null. Let's early return, instead of error since this is a harmless - // no-op - if (value === null) return this._callbackOrEmit(null, callback); - - var error = null; - if (this._doc._isInHardRollback) { - error = { - code: ERROR_CODE.ERR_DOC_IN_HARD_ROLLBACK, - message: 'Cannot submit presence. Document is processing hard rollback' - }; - } else { - error = { - code: ERROR_CODE.ERR_DOC_DOES_NOT_EXIST, - message: 'Cannot submit presence. Document has not been created' - }; - } - - return this._callbackOrEmit(error, callback); - }; - - // Record the current data state version to check if we need to transform - // the presence later - this._docDataVersionByPresenceVersion[this.presenceVersion] = this._doc._dataStateVersion; - LocalPresence.prototype.submit.call(this, value, callback); -}; - -LocalDocPresence.prototype.destroy = function(callback) { - this._emitter.removeEventListener(this._doc, 'op', this._opHandler); - this._emitter.removeEventListener(this._doc, 'create', this._createOrDelHandler); - this._emitter.removeEventListener(this._doc, 'del', this._createOrDelHandler); - this._emitter.removeEventListener(this._doc, 'load', this._loadHandler); - this._emitter.removeEventListener(this._doc, 'destroy', this._destroyHandler); - - LocalPresence.prototype.destroy.call(this, callback); -}; - -LocalDocPresence.prototype._sendPending = function() { - if (this._isSending) return; - this._isSending = true; - var presence = this; - this._doc.whenNothingPending(function() { - presence._isSending = false; - if (!presence.connection.canSend) return; - - presence._pendingMessages.forEach(function(message) { - message.t = presence._doc.type.uri; - message.v = presence._doc.version; - presence.connection.send(message); - }); - - presence._pendingMessages = []; - presence._docDataVersionByPresenceVersion = Object.create(null); - }); -}; - -LocalDocPresence.prototype._registerWithDoc = function() { - this._emitter.addEventListener(this._doc, 'op', this._opHandler); - this._emitter.addEventListener(this._doc, 'create', this._createOrDelHandler); - this._emitter.addEventListener(this._doc, 'del', this._createOrDelHandler); - this._emitter.addEventListener(this._doc, 'load', this._loadHandler); - this._emitter.addEventListener(this._doc, 'destroy', this._destroyHandler); -}; - -LocalDocPresence.prototype._transformAgainstOp = function(op, source) { - var presence = this; - var docDataVersion = this._doc._dataStateVersion; - - this._pendingMessages.forEach(function(message) { - // Check if the presence needs transforming against the op - this is to check against - // edge cases where presence is submitted from an 'op' event - var messageDocDataVersion = presence._docDataVersionByPresenceVersion[message.pv]; - if (messageDocDataVersion >= docDataVersion) return; - try { - message.p = presence._transformPresence(message.p, op, source); - // Ensure the presence's data version is kept consistent to deal with "deep" op - // submissions - presence._docDataVersionByPresenceVersion[message.pv] = docDataVersion; - } catch (error) { - var callback = presence._getCallback(message.pv); - presence._callbackOrEmit(error, callback); - } - }); - - try { - this.value = this._transformPresence(this.value, op, source); - } catch (error) { - this.emit('error', error); - } -}; - -LocalDocPresence.prototype._handleCreateOrDel = function() { - this._pendingMessages.forEach(function(message) { - message.p = null; - }); - - this.value = null; -}; - -LocalDocPresence.prototype._handleLoad = function() { - this.value = null; - this._pendingMessages = []; - this._docDataVersionByPresenceVersion = Object.create(null); -}; - -LocalDocPresence.prototype._message = function() { - var message = LocalPresence.prototype._message.call(this); - message.c = this.collection, - message.d = this.id, - message.v = null; - message.t = null; - return message; -}; - -LocalDocPresence.prototype._transformPresence = function(value, op, source) { - var type = this._doc.type; - if (!util.supportsPresence(type)) { - throw new ShareDBError( - ERROR_CODE.ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE, - 'Type does not support presence: ' + type.name - ); - } - return type.transformPresence(value, op, source); -}; diff --git a/lib/client/presence/local-presence.js b/lib/client/presence/local-presence.js deleted file mode 100644 index ae70c76ab..000000000 --- a/lib/client/presence/local-presence.js +++ /dev/null @@ -1,80 +0,0 @@ -var emitter = require('../../emitter'); -var ACTIONS = require('../../message-actions').ACTIONS; -var util = require('../../util'); - -module.exports = LocalPresence; -function LocalPresence(presence, presenceId) { - emitter.EventEmitter.call(this); - - if (!presenceId || typeof presenceId !== 'string') { - throw new Error('LocalPresence presenceId must be a string'); - } - - this.presence = presence; - this.presenceId = presenceId; - this.connection = presence.connection; - this.presenceVersion = 0; - - this.value = null; - - this._pendingMessages = []; - this._callbacksByPresenceVersion = Object.create(null); -} -emitter.mixin(LocalPresence); - -LocalPresence.prototype.submit = function(value, callback) { - this.value = value; - this.send(callback); -}; - -LocalPresence.prototype.send = function(callback) { - var message = this._message(); - this._pendingMessages.push(message); - this._callbacksByPresenceVersion[message.pv] = callback; - this._sendPending(); -}; - -LocalPresence.prototype.destroy = function(callback) { - var presence = this; - this.submit(null, function(error) { - if (error) return presence._callbackOrEmit(error, callback); - delete presence.presence.localPresences[presence.presenceId]; - if (callback) callback(); - }); -}; - -LocalPresence.prototype._sendPending = function() { - if (!this.connection.canSend) return; - var presence = this; - this._pendingMessages.forEach(function(message) { - presence.connection.send(message); - }); - - this._pendingMessages = []; -}; - -LocalPresence.prototype._ack = function(error, presenceVersion) { - var callback = this._getCallback(presenceVersion); - this._callbackOrEmit(error, callback); -}; - -LocalPresence.prototype._message = function() { - return { - a: ACTIONS.presence, - ch: this.presence.channel, - id: this.presenceId, - p: this.value, - pv: this.presenceVersion++ - }; -}; - -LocalPresence.prototype._getCallback = function(presenceVersion) { - var callback = this._callbacksByPresenceVersion[presenceVersion]; - delete this._callbacksByPresenceVersion[presenceVersion]; - return callback; -}; - -LocalPresence.prototype._callbackOrEmit = function(error, callback) { - if (callback) return util.nextTick(callback, error); - if (error) this.emit('error', error); -}; diff --git a/lib/client/presence/presence.js b/lib/client/presence/presence.js deleted file mode 100644 index 7b3fccabc..000000000 --- a/lib/client/presence/presence.js +++ /dev/null @@ -1,214 +0,0 @@ -var emitter = require('../../emitter'); -var LocalPresence = require('./local-presence'); -var RemotePresence = require('./remote-presence'); -var util = require('../../util'); -var async = require('async'); -var hat = require('hat'); -var ACTIONS = require('../../message-actions').ACTIONS; - -module.exports = Presence; -function Presence(connection, channel) { - emitter.EventEmitter.call(this); - - if (!channel || typeof channel !== 'string') { - throw new Error('Presence channel must be provided'); - } - - this.connection = connection; - this.channel = channel; - - this.wantSubscribe = false; - this.subscribed = false; - this.remotePresences = Object.create(null); - this.localPresences = Object.create(null); - - this._remotePresenceInstances = Object.create(null); - this._subscriptionCallbacksBySeq = Object.create(null); - this._wantsDestroy = false; -} -emitter.mixin(Presence); - -Presence.prototype.subscribe = function(callback) { - this._sendSubscriptionAction(true, callback); -}; - -Presence.prototype.unsubscribe = function(callback) { - this._sendSubscriptionAction(false, callback); -}; - -Presence.prototype.create = function(id) { - if (this._wantsDestroy) { - throw new Error('Presence is being destroyed'); - } - id = id || hat(); - var localPresence = this._createLocalPresence(id); - this.localPresences[id] = localPresence; - return localPresence; -}; - -Presence.prototype.destroy = function(callback) { - this._wantsDestroy = true; - var presence = this; - // Store these at the time of destruction: any LocalPresence on this - // instance at this time will be destroyed, but if the destroy is - // cancelled, any future LocalPresence objects will be kept. - // See: https://github.com/share/sharedb/pull/579 - var localIds = Object.keys(presence.localPresences); - this.unsubscribe(function(error) { - if (error) return presence._callbackOrEmit(error, callback); - var remoteIds = Object.keys(presence._remotePresenceInstances); - async.parallel( - [ - function(next) { - async.each(localIds, function(presenceId, next) { - var localPresence = presence.localPresences[presenceId]; - if (!localPresence) return next(); - localPresence.destroy(next); - }, next); - }, - function(next) { - // We don't bother stashing the RemotePresence instances because - // they're not really bound to our local state: if we want to - // destroy, we destroy them all, but if we cancel the destroy, - // we'll want to keep them all - if (!presence._wantsDestroy) return next(); - async.each(remoteIds, function(presenceId, next) { - presence._remotePresenceInstances[presenceId].destroy(next); - }, next); - } - ], - function(error) { - if (presence._wantsDestroy) delete presence.connection._presences[presence.channel]; - presence._callbackOrEmit(error, callback); - } - ); - }); -}; - -Presence.prototype._sendSubscriptionAction = function(wantSubscribe, callback) { - wantSubscribe = !!wantSubscribe; - if (wantSubscribe === this.wantSubscribe) { - if (!callback) return; - if (wantSubscribe === this.subscribed) return util.nextTick(callback); - if (Object.keys(this._subscriptionCallbacksBySeq).length) { - return this._combineSubscribeCallbackWithLastAdded(callback); - } - } - this.wantSubscribe = wantSubscribe; - var action = this.wantSubscribe ? ACTIONS.presenceSubscribe : ACTIONS.presenceUnsubscribe; - var seq = this.connection._presenceSeq++; - this._subscriptionCallbacksBySeq[seq] = callback; - if (this.connection.canSend) { - this.connection._sendPresenceAction(action, seq, this); - } -}; - -Presence.prototype._requestRemotePresence = function() { - this.connection._requestRemotePresence(this.channel); -}; - -Presence.prototype._handleSubscribe = function(error, seq) { - if (this.wantSubscribe) this.subscribed = true; - var callback = this._subscriptionCallback(seq); - this._callbackOrEmit(error, callback); -}; - -Presence.prototype._handleUnsubscribe = function(error, seq) { - this.subscribed = false; - var callback = this._subscriptionCallback(seq); - this._callbackOrEmit(error, callback); -}; - -Presence.prototype._receiveUpdate = function(error, message) { - var localPresence = util.dig(this.localPresences, message.id); - if (localPresence) return localPresence._ack(error, message.pv); - - if (error) return this.emit('error', error); - var presence = this; - var remotePresence = util.digOrCreate(this._remotePresenceInstances, message.id, function() { - return presence._createRemotePresence(message.id); - }); - - remotePresence.receiveUpdate(message); -}; - -Presence.prototype._updateRemotePresence = function(remotePresence) { - this.remotePresences[remotePresence.presenceId] = remotePresence.value; - if (remotePresence.value === null) this._removeRemotePresence(remotePresence.presenceId); - this.emit('receive', remotePresence.presenceId, remotePresence.value); -}; - -Presence.prototype._broadcastAllLocalPresence = function(error) { - if (error) return this.emit('error', error); - for (var id in this.localPresences) { - var localPresence = this.localPresences[id]; - if (localPresence.value !== null) localPresence.send(); - } -}; - -Presence.prototype._removeRemotePresence = function(id) { - this._remotePresenceInstances[id].destroy(); - delete this._remotePresenceInstances[id]; - delete this.remotePresences[id]; -}; - -Presence.prototype._onConnectionStateChanged = function() { - if (!this.connection.canSend) { - this.subscribed = false; - return; - } - this._resubscribe(); - for (var id in this.localPresences) { - this.localPresences[id]._sendPending(); - } -}; - -Presence.prototype._resubscribe = function() { - var callbacks = []; - for (var seq in this._subscriptionCallbacksBySeq) { - var callback = this._subscriptionCallback(seq); - callbacks.push(callback); - } - - if (!this.wantSubscribe) return this._callEachOrEmit(callbacks); - - var presence = this; - this.subscribe(function(error) { - presence._callEachOrEmit(callbacks, error); - }); -}; - -Presence.prototype._subscriptionCallback = function(seq) { - var callback = this._subscriptionCallbacksBySeq[seq]; - delete this._subscriptionCallbacksBySeq[seq]; - return callback; -}; - -Presence.prototype._callbackOrEmit = function(error, callback) { - if (callback) return util.nextTick(callback, error); - if (error) this.emit('error', error); -}; - -Presence.prototype._createLocalPresence = function(id) { - return new LocalPresence(this, id); -}; - -Presence.prototype._createRemotePresence = function(id) { - return new RemotePresence(this, id); -}; - -Presence.prototype._callEachOrEmit = function(callbacks, error) { - var called = util.callEach(callbacks, error); - if (!called && error) this.emit('error', error); -}; - -Presence.prototype._combineSubscribeCallbackWithLastAdded = function(callback) { - var seqs = Object.keys(this._subscriptionCallbacksBySeq); - var lastSeq = seqs[seqs.length - 1]; - var originalCallback = this._subscriptionCallbacksBySeq[lastSeq]; - if (!originalCallback) return this._subscriptionCallbacksBySeq[lastSeq] = callback; - this._subscriptionCallbacksBySeq[lastSeq] = function(error) { - originalCallback(error); - callback(error); - }; -}; diff --git a/lib/client/presence/remote-doc-presence.js b/lib/client/presence/remote-doc-presence.js deleted file mode 100644 index 0f3b7f74c..000000000 --- a/lib/client/presence/remote-doc-presence.js +++ /dev/null @@ -1,150 +0,0 @@ -var RemotePresence = require('./remote-presence'); -var ot = require('../../ot'); - -module.exports = RemoteDocPresence; -function RemoteDocPresence(presence, presenceId) { - RemotePresence.call(this, presence, presenceId); - - this.collection = this.presence.collection; - this.id = this.presence.id; - this.src = null; - this.presenceVersion = null; - - this._doc = this.connection.get(this.collection, this.id); - this._emitter = this.connection._docPresenceEmitter; - this._pending = null; - this._opCache = null; - this._pendingSetPending = false; - - this._opHandler = this._handleOp.bind(this); - this._createDelHandler = this._handleCreateDel.bind(this); - this._loadHandler = this._handleLoad.bind(this); - this._registerWithDoc(); -} - -RemoteDocPresence.prototype = Object.create(RemotePresence.prototype); - -RemoteDocPresence.prototype.receiveUpdate = function(message) { - if (this._pending && message.pv < this._pending.pv) return; - this.src = message.src; - this._pending = message; - this._setPendingPresence(); -}; - -RemoteDocPresence.prototype.destroy = function(callback) { - this._emitter.removeEventListener(this._doc, 'op', this._opHandler); - this._emitter.removeEventListener(this._doc, 'create', this._createDelHandler); - this._emitter.removeEventListener(this._doc, 'del', this._createDelHandler); - this._emitter.removeEventListener(this._doc, 'load', this._loadHandler); - - RemotePresence.prototype.destroy.call(this, callback); -}; - -RemoteDocPresence.prototype._registerWithDoc = function() { - this._emitter.addEventListener(this._doc, 'op', this._opHandler); - this._emitter.addEventListener(this._doc, 'create', this._createDelHandler); - this._emitter.addEventListener(this._doc, 'del', this._createDelHandler); - this._emitter.addEventListener(this._doc, 'load', this._loadHandler); -}; - -RemoteDocPresence.prototype._setPendingPresence = function() { - if (this._pendingSetPending) return; - this._pendingSetPending = true; - var presence = this; - this._doc.whenNothingPending(function() { - presence._pendingSetPending = false; - if (!presence._pending) return; - if (presence._pending.pv < presence.presenceVersion) return presence._pending = null; - - if (presence._pending.v > presence._doc.version) { - return presence._doc.fetch(); - } - - if (!presence._catchUpStalePresence()) return; - - presence.value = presence._pending.p; - presence.presenceVersion = presence._pending.pv; - presence._pending = null; - presence.presence._updateRemotePresence(presence); - }); -}; - -RemoteDocPresence.prototype._handleOp = function(op, source, connectionId) { - var isOwnOp = connectionId === this.src; - this._transformAgainstOp(op, isOwnOp); - this._cacheOp(op, isOwnOp); - this._setPendingPresence(); -}; - -RemotePresence.prototype._handleCreateDel = function() { - this._cacheOp(null); - this._setPendingPresence(); -}; - -RemotePresence.prototype._handleLoad = function() { - this.value = null; - this._pending = null; - this._opCache = null; - this.presence._updateRemotePresence(this); -}; - -RemoteDocPresence.prototype._transformAgainstOp = function(op, isOwnOp) { - if (!this.value) return; - - try { - this.value = this._doc.type.transformPresence(this.value, op, isOwnOp); - } catch (error) { - return this.presence.emit('error', error); - } - this.presence._updateRemotePresence(this); -}; - -RemoteDocPresence.prototype._catchUpStalePresence = function() { - if (this._pending.v >= this._doc.version) return true; - - if (!this._opCache) { - this._startCachingOps(); - this._doc.fetch(); - this.presence._requestRemotePresence(); - return false; - } - - while (this._opCache[this._pending.v]) { - var item = this._opCache[this._pending.v]; - var op = item.op; - var isOwnOp = item.isOwnOp; - // We use a null op to signify a create or a delete operation. In both - // cases we just want to reset the presence (which doesn't make sense - // in a new document), so just set the presence to null. - if (op === null) { - this._pending.p = null; - this._pending.v++; - } else { - ot.transformPresence(this._pending, op, isOwnOp); - } - } - - var hasCaughtUp = this._pending.v >= this._doc.version; - if (hasCaughtUp) { - this._stopCachingOps(); - } - - return hasCaughtUp; -}; - -RemoteDocPresence.prototype._startCachingOps = function() { - this._opCache = []; -}; - -RemoteDocPresence.prototype._stopCachingOps = function() { - this._opCache = null; -}; - -RemoteDocPresence.prototype._cacheOp = function(op, isOwnOp) { - if (this._opCache) { - op = op ? {op: op} : null; - // Subtract 1 from the current doc version, because an op with v3 - // should be read as the op that takes a doc from v3 -> v4 - this._opCache[this._doc.version - 1] = {op: op, isOwnOp: isOwnOp}; - } -}; diff --git a/lib/client/presence/remote-presence.js b/lib/client/presence/remote-presence.js deleted file mode 100644 index e88a15d9d..000000000 --- a/lib/client/presence/remote-presence.js +++ /dev/null @@ -1,24 +0,0 @@ -var util = require('../../util'); - -module.exports = RemotePresence; -function RemotePresence(presence, presenceId) { - this.presence = presence; - this.presenceId = presenceId; - this.connection = this.presence.connection; - - this.value = null; - this.presenceVersion = 0; -} - -RemotePresence.prototype.receiveUpdate = function(message) { - if (message.pv < this.presenceVersion) return; - this.value = message.p; - this.presenceVersion = message.pv; - this.presence._updateRemotePresence(this); -}; - -RemotePresence.prototype.destroy = function(callback) { - delete this.presence._remotePresenceInstances[this.presenceId]; - delete this.presence.remotePresences[this.presenceId]; - if (callback) util.nextTick(callback); -}; diff --git a/lib/client/query.js b/lib/client/query.js deleted file mode 100644 index a8665dc8a..000000000 --- a/lib/client/query.js +++ /dev/null @@ -1,200 +0,0 @@ -var emitter = require('../emitter'); -var ACTIONS = require('../message-actions').ACTIONS; -var util = require('../util'); - -// Queries are live requests to the database for particular sets of fields. -// -// The server actively tells the client when there's new data that matches -// a set of conditions. -module.exports = Query; -function Query(action, connection, id, collection, query, options, callback) { - emitter.EventEmitter.call(this); - - // 'qf' or 'qs' - this.action = action; - - this.connection = connection; - this.id = id; - this.collection = collection; - - // The query itself. For mongo, this should look something like {"data.x":5} - this.query = query; - - // A list of resulting documents. These are actual documents, complete with - // data and all the rest. It is possible to pass in an initial results set, - // so that a query can be serialized and then re-established - this.results = null; - if (options && options.results) { - this.results = options.results; - delete options.results; - } - this.extra = undefined; - - // Options to pass through with the query - this.options = options; - - this.callback = callback; - this.ready = false; - this.sent = false; -} -emitter.mixin(Query); - -Query.prototype.hasPending = function() { - return !this.ready; -}; - -// Helper for subscribe & fetch, since they share the same message format. -// -// This function actually issues the query. -Query.prototype.send = function() { - if (!this.connection.canSend) return; - - var message = { - a: this.action, - id: this.id, - c: this.collection, - q: this.query - }; - if (this.options) { - message.o = this.options; - } - if (this.results) { - // Collect the version of all the documents in the current result set so we - // don't need to be sent their snapshots again. - var results = []; - for (var i = 0; i < this.results.length; i++) { - var doc = this.results[i]; - results.push([doc.id, doc.version]); - } - message.r = results; - } - - this.connection.send(message); - this.sent = true; -}; - -// Destroy the query object. Any subsequent messages for the query will be -// ignored by the connection. -Query.prototype.destroy = function(callback) { - if (this.connection.canSend && this.action === ACTIONS.querySubscribe) { - this.connection.send({a: ACTIONS.queryUnsubscribe, id: this.id}); - } - this.connection._destroyQuery(this); - // There is a callback for consistency, but we don't actually wait for the - // server's unsubscribe message currently - if (callback) util.nextTick(callback); -}; - -Query.prototype._onConnectionStateChanged = function() { - if (this.connection.canSend && !this.sent) { - this.send(); - } else { - this.sent = false; - } -}; - -Query.prototype._handleFetch = function(err, data, extra) { - // Once a fetch query gets its data, it is destroyed. - this.connection._destroyQuery(this); - this._handleResponse(err, data, extra); -}; - -Query.prototype._handleSubscribe = function(err, data, extra) { - this._handleResponse(err, data, extra); -}; - -Query.prototype._handleResponse = function(err, data, extra) { - var callback = this.callback; - this.callback = null; - if (err) return this._finishResponse(err, callback); - if (!data) return this._finishResponse(null, callback); - - var query = this; - var wait = 1; - var finish = function(err) { - if (err) return query._finishResponse(err, callback); - if (--wait) return; - query._finishResponse(null, callback); - }; - - if (Array.isArray(data)) { - wait += data.length; - this.results = this._ingestSnapshots(data, finish); - this.extra = extra; - } else { - for (var id in data) { - wait++; - var snapshot = data[id]; - var doc = this.connection.get(snapshot.c || this.collection, id); - doc.ingestSnapshot(snapshot, finish); - } - } - - finish(); -}; - -Query.prototype._ingestSnapshots = function(snapshots, finish) { - var results = []; - for (var i = 0; i < snapshots.length; i++) { - var snapshot = snapshots[i]; - var doc = this.connection.get(snapshot.c || this.collection, snapshot.d); - doc.ingestSnapshot(snapshot, finish); - results.push(doc); - } - return results; -}; - -Query.prototype._finishResponse = function(err, callback) { - this.emit('ready'); - this.ready = true; - if (err) { - this.connection._destroyQuery(this); - if (callback) return callback(err); - return this.emit('error', err); - } - if (callback) callback(null, this.results, this.extra); -}; - -Query.prototype._handleError = function(err) { - this.emit('error', err); -}; - -Query.prototype._handleDiff = function(diff) { - // We need to go through the list twice. First, we'll ingest all the new - // documents. After that we'll emit events and actually update our list. - // This avoids race conditions around setting documents to be subscribed & - // unsubscribing documents in event callbacks. - for (var i = 0; i < diff.length; i++) { - var d = diff[i]; - if (d.type === 'insert') d.values = this._ingestSnapshots(d.values); - } - - for (var i = 0; i < diff.length; i++) { - var d = diff[i]; - switch (d.type) { - case 'insert': - var newDocs = d.values; - Array.prototype.splice.apply(this.results, [d.index, 0].concat(newDocs)); - this.emit('insert', newDocs, d.index); - break; - case 'remove': - var howMany = d.howMany || 1; - var removed = this.results.splice(d.index, howMany); - this.emit('remove', removed, d.index); - break; - case 'move': - var howMany = d.howMany || 1; - var docs = this.results.splice(d.from, howMany); - Array.prototype.splice.apply(this.results, [d.to, 0].concat(docs)); - this.emit('move', docs, d.from, d.to); - break; - } - } - - this.emit('changed', this.results); -}; - -Query.prototype._handleExtra = function(extra) { - this.extra = extra; - this.emit('extra', extra); -}; diff --git a/lib/client/snapshot-request/snapshot-request.js b/lib/client/snapshot-request/snapshot-request.js deleted file mode 100644 index 95f68055b..000000000 --- a/lib/client/snapshot-request/snapshot-request.js +++ /dev/null @@ -1,54 +0,0 @@ -var Snapshot = require('../../snapshot'); -var emitter = require('../../emitter'); - -module.exports = SnapshotRequest; - -function SnapshotRequest(connection, requestId, collection, id, callback) { - emitter.EventEmitter.call(this); - - if (typeof callback !== 'function') { - throw new Error('Callback is required for SnapshotRequest'); - } - - this.requestId = requestId; - this.connection = connection; - this.id = id; - this.collection = collection; - this.callback = callback; - - this.sent = false; -} -emitter.mixin(SnapshotRequest); - -SnapshotRequest.prototype.send = function() { - if (!this.connection.canSend) { - return; - } - - this.connection.send(this._message()); - this.sent = true; -}; - -SnapshotRequest.prototype._onConnectionStateChanged = function() { - if (this.connection.canSend) { - if (!this.sent) this.send(); - } else { - // If the connection can't send, then we've had a disconnection, and even if we've already sent - // the request previously, we need to re-send it over this reconnected client, so reset the - // sent flag to false. - this.sent = false; - } -}; - -SnapshotRequest.prototype._handleResponse = function(error, message) { - this.emit('ready'); - - if (error) { - return this.callback(error); - } - - var metadata = message.meta ? message.meta : null; - var snapshot = new Snapshot(this.id, message.v, message.type, message.data, metadata); - - this.callback(null, snapshot); -}; diff --git a/lib/client/snapshot-request/snapshot-timestamp-request.js b/lib/client/snapshot-request/snapshot-timestamp-request.js deleted file mode 100644 index 8d4cec70d..000000000 --- a/lib/client/snapshot-request/snapshot-timestamp-request.js +++ /dev/null @@ -1,27 +0,0 @@ -var SnapshotRequest = require('./snapshot-request'); -var util = require('../../util'); -var ACTIONS = require('../../message-actions').ACTIONS; - -module.exports = SnapshotTimestampRequest; - -function SnapshotTimestampRequest(connection, requestId, collection, id, timestamp, callback) { - SnapshotRequest.call(this, connection, requestId, collection, id, callback); - - if (!util.isValidTimestamp(timestamp)) { - throw new Error('Snapshot timestamp must be a positive integer or null'); - } - - this.timestamp = timestamp; -} - -SnapshotTimestampRequest.prototype = Object.create(SnapshotRequest.prototype); - -SnapshotTimestampRequest.prototype._message = function() { - return { - a: ACTIONS.snapshotFetchByTimestamp, - id: this.requestId, - c: this.collection, - d: this.id, - ts: this.timestamp - }; -}; diff --git a/lib/client/snapshot-request/snapshot-version-request.js b/lib/client/snapshot-request/snapshot-version-request.js deleted file mode 100644 index 369bf82ee..000000000 --- a/lib/client/snapshot-request/snapshot-version-request.js +++ /dev/null @@ -1,27 +0,0 @@ -var SnapshotRequest = require('./snapshot-request'); -var util = require('../../util'); -var ACTIONS = require('../../message-actions').ACTIONS; - -module.exports = SnapshotVersionRequest; - -function SnapshotVersionRequest(connection, requestId, collection, id, version, callback) { - SnapshotRequest.call(this, connection, requestId, collection, id, callback); - - if (!util.isValidVersion(version)) { - throw new Error('Snapshot version must be a positive integer or null'); - } - - this.version = version; -} - -SnapshotVersionRequest.prototype = Object.create(SnapshotRequest.prototype); - -SnapshotVersionRequest.prototype._message = function() { - return { - a: ACTIONS.snapshotFetch, - id: this.requestId, - c: this.collection, - d: this.id, - v: this.version - }; -}; diff --git a/lib/db/index.js b/lib/db/index.js deleted file mode 100644 index 043276bf4..000000000 --- a/lib/db/index.js +++ /dev/null @@ -1,107 +0,0 @@ -var async = require('async'); -var ShareDBError = require('../error'); - -var ERROR_CODE = ShareDBError.CODES; - -function DB(options) { - // pollDebounce is the minimum time in ms between query polls - this.pollDebounce = options && options.pollDebounce; -} -module.exports = DB; - -// When false, Backend will handle projections instead of DB -DB.prototype.projectsSnapshots = false; -DB.prototype.disableSubscribe = false; - -DB.prototype.close = function(callback) { - if (callback) callback(); -}; - -DB.prototype.commit = function(collection, id, op, snapshot, options, callback) { - callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, 'commit DB method unimplemented')); -}; - -DB.prototype.getSnapshot = function(collection, id, fields, options, callback) { - callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, 'getSnapshot DB method unimplemented')); -}; - -DB.prototype.getSnapshotBulk = function(collection, ids, fields, options, callback) { - var results = Object.create(null); - var db = this; - async.each(ids, function(id, eachCb) { - db.getSnapshot(collection, id, fields, options, function(err, snapshot) { - if (err) return eachCb(err); - results[id] = snapshot; - eachCb(); - }); - }, function(err) { - if (err) return callback(err); - callback(null, results); - }); -}; - -DB.prototype.getOps = function(collection, id, from, to, options, callback) { - callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, 'getOps DB method unimplemented')); -}; - -DB.prototype.getOpsToSnapshot = function(collection, id, from, snapshot, options, callback) { - var to = snapshot.v; - this.getOps(collection, id, from, to, options, callback); -}; - -DB.prototype.getOpsBulk = function(collection, fromMap, toMap, options, callback) { - var results = Object.create(null); - var db = this; - async.forEachOf(fromMap, function(from, id, eachCb) { - var to = toMap && toMap[id]; - db.getOps(collection, id, from, to, options, function(err, ops) { - if (err) return eachCb(err); - results[id] = ops; - eachCb(); - }); - }, function(err) { - if (err) return callback(err); - callback(null, results); - }); -}; - -DB.prototype.getCommittedOpVersion = function(collection, id, snapshot, op, options, callback) { - this.getOpsToSnapshot(collection, id, 0, snapshot, options, function(err, ops) { - if (err) return callback(err); - for (var i = ops.length; i--;) { - var item = ops[i]; - if (op.src === item.src && op.seq === item.seq) { - return callback(null, item.v); - } - } - callback(); - }); -}; - -DB.prototype.query = function(collection, query, fields, options, callback) { - callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, 'query DB method unimplemented')); -}; - -DB.prototype.queryPoll = function(collection, query, options, callback) { - var fields = Object.create(null); - this.query(collection, query, fields, options, function(err, snapshots, extra) { - if (err) return callback(err); - var ids = []; - for (var i = 0; i < snapshots.length; i++) { - ids.push(snapshots[i].id); - } - callback(null, ids, extra); - }); -}; - -DB.prototype.queryPollDoc = function(collection, id, query, options, callback) { - callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, 'queryPollDoc DB method unimplemented')); -}; - -DB.prototype.canPollDoc = function() { - return false; -}; - -DB.prototype.skipPoll = function() { - return false; -}; diff --git a/lib/db/memory.js b/lib/db/memory.js deleted file mode 100644 index 2db274d6e..000000000 --- a/lib/db/memory.js +++ /dev/null @@ -1,189 +0,0 @@ -var DB = require('./index'); -var Snapshot = require('../snapshot'); -var util = require('../util'); -var clone = util.clone; - -// In-memory ShareDB database -// -// This adapter is not appropriate for production use. It is intended for -// testing and as an API example for people implementing database adaptors. It -// is fully functional, except it stores all documents & operations forever in -// memory. As such, memory usage will grow without bound, it doesn't scale -// across multiple node processes and you'll lose all your data if the server -// restarts. Query APIs are adapter specific. Use with care. - -function MemoryDB(options) { - if (!(this instanceof MemoryDB)) return new MemoryDB(options); - DB.call(this, options); - - // Map from collection name -> doc id -> doc snapshot ({v:, type:, data:}) - this.docs = Object.create(null); - - // Map from collection name -> doc id -> list of operations. Operations - // don't store their version - instead their version is simply the index in - // the list. - this.ops = Object.create(null); - - this.closed = false; -}; -module.exports = MemoryDB; - -MemoryDB.prototype = Object.create(DB.prototype); - -MemoryDB.prototype.close = function(callback) { - this.closed = true; - if (callback) callback(); -}; - -// Persists an op and snapshot if it is for the next version. Calls back with -// callback(err, succeeded) -MemoryDB.prototype.commit = function(collection, id, op, snapshot, options, callback) { - var db = this; - if (typeof callback !== 'function') throw new Error('Callback required'); - util.nextTick(function() { - var version = db._getVersionSync(collection, id); - if (snapshot.v !== version + 1) { - var succeeded = false; - return callback(null, succeeded); - } - var err = db._writeOpSync(collection, id, op); - if (err) return callback(err); - err = db._writeSnapshotSync(collection, id, snapshot); - if (err) return callback(err); - - var succeeded = true; - callback(null, succeeded); - }); -}; - -// Get the named document from the database. The callback is called with (err, -// snapshot). A snapshot with a version of zero is returned if the docuemnt -// has never been created in the database. -MemoryDB.prototype.getSnapshot = function(collection, id, fields, options, callback) { - var includeMetadata = (fields && fields.$submit) || (options && options.metadata); - var db = this; - if (typeof callback !== 'function') throw new Error('Callback required'); - util.nextTick(function() { - var snapshot = db._getSnapshotSync(collection, id, includeMetadata); - callback(null, snapshot); - }); -}; - -// Get operations between [from, to) noninclusively. (Ie, the range should -// contain start but not end). -// -// If end is null, this function should return all operations from start onwards. -// -// The operations that getOps returns don't need to have a version: field. -// The version will be inferred from the parameters if it is missing. -// -// Callback should be called as callback(error, [list of ops]); -MemoryDB.prototype.getOps = function(collection, id, from, to, options, callback) { - var includeMetadata = options && options.metadata; - var db = this; - if (typeof callback !== 'function') throw new Error('Callback required'); - util.nextTick(function() { - var opLog = db._getOpLogSync(collection, id); - if (!from) from = 0; - if (to == null) to = opLog.length; - var ops = clone(opLog.slice(from, to).filter(Boolean)); - if (ops.length < to - from) { - return callback(new Error('Missing ops')); - } - if (!includeMetadata) { - for (var i = 0; i < ops.length; i++) { - delete ops[i].m; - } - } - callback(null, ops); - }); -}; - -MemoryDB.prototype.deleteOps = function(collection, id, from, to, options, callback) { - if (typeof callback !== 'function') throw new Error('Callback required'); - var db = this; - util.nextTick(function() { - var opLog = db._getOpLogSync(collection, id); - if (!from) from = 0; - if (to == null) to = opLog.length; - for (var i = from; i < to; i++) opLog[i] = null; - callback(null); - }); -}; - -// The memory database query function returns all documents in a collection -// regardless of query by default -MemoryDB.prototype.query = function(collection, query, fields, options, callback) { - var includeMetadata = options && options.metadata; - var db = this; - if (typeof callback !== 'function') throw new Error('Callback required'); - util.nextTick(function() { - var collectionDocs = db.docs[collection]; - var snapshots = []; - for (var id in collectionDocs || {}) { - var snapshot = db._getSnapshotSync(collection, id, includeMetadata); - snapshots.push(snapshot); - } - try { - var result = db._querySync(snapshots, query, options); - callback(null, result.snapshots, result.extra); - } catch (err) { - callback(err); - } - }); -}; - -// For testing, it may be useful to implement the desired query -// language by defining this function. Returns an object with -// two properties: -// - snapshots: array of query result snapshots -// - extra: (optional) other types of results, such as counts -MemoryDB.prototype._querySync = function(snapshots) { - return {snapshots: snapshots}; -}; - -MemoryDB.prototype._writeOpSync = function(collection, id, op) { - var opLog = this._getOpLogSync(collection, id); - // This will write an op in the log at its version, which should always be - // the next item in the array under normal operation - opLog[op.v] = clone(op); -}; - -// Create, update, and delete snapshots. For creates and updates, a snapshot -// object will be passed in with a type property. If there is no type property, -// it should be considered a delete -MemoryDB.prototype._writeSnapshotSync = function(collection, id, snapshot) { - var collectionDocs = this.docs[collection] || (this.docs[collection] = Object.create(null)); - if (!snapshot.type) { - delete collectionDocs[id]; - } else { - collectionDocs[id] = clone(snapshot); - } -}; - -MemoryDB.prototype._getSnapshotSync = function(collection, id, includeMetadata) { - var collectionDocs = this.docs[collection]; - // We need to clone the snapshot, because ShareDB assumes each call to - // getSnapshot returns a new object - var doc = collectionDocs && collectionDocs[id]; - var snapshot; - if (doc) { - var data = clone(doc.data); - var meta = (includeMetadata) ? clone(doc.m) : null; - snapshot = new Snapshot(id, doc.v, doc.type, data, meta); - } else { - var version = this._getVersionSync(collection, id); - snapshot = new Snapshot(id, version, null, undefined, null); - } - return snapshot; -}; - -MemoryDB.prototype._getOpLogSync = function(collection, id) { - var collectionOps = this.ops[collection] || (this.ops[collection] = Object.create(null)); - return collectionOps[id] || (collectionOps[id] = []); -}; - -MemoryDB.prototype._getVersionSync = function(collection, id) { - var collectionOps = this.ops[collection]; - return (collectionOps && collectionOps[id] && collectionOps[id].length) || 0; -}; diff --git a/lib/error.js b/lib/error.js deleted file mode 100644 index 85750104d..000000000 --- a/lib/error.js +++ /dev/null @@ -1,82 +0,0 @@ -function ShareDBError(code, message) { - this.code = code; - this.message = message || ''; - if (Error.captureStackTrace) { - Error.captureStackTrace(this, ShareDBError); - } else { - this.stack = new Error().stack; - } -} - -ShareDBError.prototype = Object.create(Error.prototype); -ShareDBError.prototype.constructor = ShareDBError; -ShareDBError.prototype.name = 'ShareDBError'; - -ShareDBError.CODES = { - ERR_APPLY_OP_VERSION_DOES_NOT_MATCH_SNAPSHOT: 'ERR_APPLY_OP_VERSION_DOES_NOT_MATCH_SNAPSHOT', - ERR_APPLY_SNAPSHOT_NOT_PROVIDED: 'ERR_APPLY_SNAPSHOT_NOT_PROVIDED', - ERR_FIXUP_IS_ONLY_VALID_ON_APPLY: 'ERR_FIXUP_IS_ONLY_VALID_ON_APPLY', - ERR_CANNOT_FIXUP_DELETION: 'ERR_CANNOT_FIXUP_DELETION', - ERR_CLIENT_ID_BADLY_FORMED: 'ERR_CLIENT_ID_BADLY_FORMED', - ERR_CANNOT_PING_OFFLINE: 'ERR_CANNOT_PING_OFFLINE', - ERR_CONNECTION_SEQ_INTEGER_OVERFLOW: 'ERR_CONNECTION_SEQ_INTEGER_OVERFLOW', - ERR_CONNECTION_STATE_TRANSITION_INVALID: 'ERR_CONNECTION_STATE_TRANSITION_INVALID', - ERR_DATABASE_ADAPTER_NOT_FOUND: 'ERR_DATABASE_ADAPTER_NOT_FOUND', - ERR_DATABASE_DOES_NOT_SUPPORT_SUBSCRIBE: 'ERR_DATABASE_DOES_NOT_SUPPORT_SUBSCRIBE', - ERR_DATABASE_METHOD_NOT_IMPLEMENTED: 'ERR_DATABASE_METHOD_NOT_IMPLEMENTED', - ERR_DEFAULT_TYPE_MISMATCH: 'ERR_DEFAULT_TYPE_MISMATCH', - ERR_DOC_MISSING_VERSION: 'ERR_DOC_MISSING_VERSION', - ERR_DOC_ALREADY_CREATED: 'ERR_DOC_ALREADY_CREATED', - ERR_DOC_DOES_NOT_EXIST: 'ERR_DOC_DOES_NOT_EXIST', - ERR_DOC_TYPE_NOT_RECOGNIZED: 'ERR_DOC_TYPE_NOT_RECOGNIZED', - ERR_DOC_WAS_DELETED: 'ERR_DOC_WAS_DELETED', - ERR_DOC_IN_HARD_ROLLBACK: 'ERR_DOC_IN_HARD_ROLLBACK', - ERR_INFLIGHT_OP_MISSING: 'ERR_INFLIGHT_OP_MISSING', - ERR_INGESTED_SNAPSHOT_HAS_NO_VERSION: 'ERR_INGESTED_SNAPSHOT_HAS_NO_VERSION', - ERR_MAX_SUBMIT_RETRIES_EXCEEDED: 'ERR_MAX_SUBMIT_RETRIES_EXCEEDED', - ERR_MESSAGE_BADLY_FORMED: 'ERR_MESSAGE_BADLY_FORMED', - ERR_MILESTONE_ARGUMENT_INVALID: 'ERR_MILESTONE_ARGUMENT_INVALID', - ERR_NO_OP: 'ERR_NO_OP', - ERR_OP_ALREADY_SUBMITTED: 'ERR_OP_ALREADY_SUBMITTED', - ERR_OP_NOT_ALLOWED_IN_PROJECTION: 'ERR_OP_NOT_ALLOWED_IN_PROJECTION', - ERR_OP_SUBMIT_REJECTED: 'ERR_OP_SUBMIT_REJECTED', - ERR_PENDING_OP_REMOVED_BY_OP_SUBMIT_REJECTED: 'ERR_PENDING_OP_REMOVED_BY_OP_SUBMIT_REJECTED', - ERR_HARD_ROLLBACK_FETCH_FAILED: 'ERR_HARD_ROLLBACK_FETCH_FAILED', - ERR_OP_VERSION_MISMATCH_AFTER_TRANSFORM: 'ERR_OP_VERSION_MISMATCH_AFTER_TRANSFORM', - ERR_OP_VERSION_MISMATCH_DURING_TRANSFORM: 'ERR_OP_VERSION_MISMATCH_DURING_TRANSFORM', - ERR_OP_VERSION_NEWER_THAN_CURRENT_SNAPSHOT: 'ERR_OP_VERSION_NEWER_THAN_CURRENT_SNAPSHOT', - ERR_OT_LEGACY_JSON0_OP_CANNOT_BE_NORMALIZED: 'ERR_OT_LEGACY_JSON0_OP_CANNOT_BE_NORMALIZED', - ERR_OT_OP_BADLY_FORMED: 'ERR_OT_OP_BADLY_FORMED', - ERR_OT_OP_NOT_APPLIED: 'ERR_OT_OP_NOT_APPLIED', - ERR_OT_OP_NOT_PROVIDED: 'ERR_OT_OP_NOT_PROVIDED', - ERR_PRESENCE_TRANSFORM_FAILED: 'ERR_PRESENCE_TRANSFORM_FAILED', - ERR_PROTOCOL_VERSION_NOT_SUPPORTED: 'ERR_PROTOCOL_VERSION_NOT_SUPPORTED', - ERR_QUERY_CHANNEL_MISSING: 'ERR_QUERY_CHANNEL_MISSING', - ERR_QUERY_EMITTER_LISTENER_NOT_ASSIGNED: 'ERR_QUERY_EMITTER_LISTENER_NOT_ASSIGNED', - /** - * A special error that a "readSnapshots" middleware implementation can use to indicate that it - * wishes for the ShareDB client to treat it as a silent rejection, not passing the error back to - * user code. - * - * For subscribes, the ShareDB client will still cancel the document subscription. - */ - ERR_SNAPSHOT_READ_SILENT_REJECTION: 'ERR_SNAPSHOT_READ_SILENT_REJECTION', - /** - * A "readSnapshots" middleware rejected the reads of specific snapshots. - * - * This error code is mostly for server use and generally will not be encountered on the client. - * Instead, each specific doc that encountered an error will receive its specific error. - * - * The one exception is for queries, where a "readSnapshots" rejection of specific snapshots will - * cause the client to receive this error for the whole query, since queries don't support - * doc-specific errors. - */ - ERR_SNAPSHOT_READS_REJECTED: 'ERR_SNAPSHOT_READS_REJECTED', - ERR_SUBMIT_TRANSFORM_OPS_NOT_FOUND: 'ERR_SUBMIT_TRANSFORM_OPS_NOT_FOUND', - ERR_TYPE_CANNOT_BE_PROJECTED: 'ERR_TYPE_CANNOT_BE_PROJECTED', - ERR_TYPE_DOES_NOT_SUPPORT_COMPOSE: 'ERR_TYPE_DOES_NOT_SUPPORT_COMPOSE', - ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE: 'ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE', - ERR_UNKNOWN_ERROR: 'ERR_UNKNOWN_ERROR' -}; - -module.exports = ShareDBError; diff --git a/lib/index.js b/lib/index.js deleted file mode 100644 index 46fd974c7..000000000 --- a/lib/index.js +++ /dev/null @@ -1,19 +0,0 @@ -var Backend = require('./backend'); -module.exports = Backend; - -Backend.Agent = require('./agent'); -Backend.Backend = Backend; -Backend.DB = require('./db'); -Backend.Error = require('./error'); -Backend.logger = require('./logger'); -Backend.MemoryDB = require('./db/memory'); -Backend.MemoryMilestoneDB = require('./milestone-db/memory'); -Backend.MemoryPubSub = require('./pubsub/memory'); -Backend.MESSAGE_ACTIONS = require('./message-actions').ACTIONS; -Backend.MilestoneDB = require('./milestone-db'); -Backend.ot = require('./ot'); -Backend.projections = require('./projections'); -Backend.PubSub = require('./pubsub'); -Backend.QueryEmitter = require('./query-emitter'); -Backend.SubmitRequest = require('./submit-request'); -Backend.types = require('./types'); diff --git a/lib/logger/index.js b/lib/logger/index.js deleted file mode 100644 index 9a80c1f1c..000000000 --- a/lib/logger/index.js +++ /dev/null @@ -1,3 +0,0 @@ -var Logger = require('./logger'); -var logger = new Logger(); -module.exports = logger; diff --git a/lib/logger/logger.js b/lib/logger/logger.js deleted file mode 100644 index 2e4858798..000000000 --- a/lib/logger/logger.js +++ /dev/null @@ -1,26 +0,0 @@ -var SUPPORTED_METHODS = [ - 'info', - 'warn', - 'error' -]; - -function Logger() { - var defaultMethods = Object.create(null); - SUPPORTED_METHODS.forEach(function(method) { - // Deal with Chrome issue: https://bugs.chromium.org/p/chromium/issues/detail?id=179628 - defaultMethods[method] = console[method].bind(console); - }); - this.setMethods(defaultMethods); -} -module.exports = Logger; - -Logger.prototype.setMethods = function(overrides) { - overrides = overrides || {}; - var logger = this; - - SUPPORTED_METHODS.forEach(function(method) { - if (typeof overrides[method] === 'function') { - logger[method] = overrides[method]; - } - }); -}; diff --git a/lib/milestone-db/index.js b/lib/milestone-db/index.js deleted file mode 100644 index 826e20bdb..000000000 --- a/lib/milestone-db/index.js +++ /dev/null @@ -1,78 +0,0 @@ -var emitter = require('../emitter'); -var ShareDBError = require('../error'); -var util = require('../util'); - -var ERROR_CODE = ShareDBError.CODES; - -module.exports = MilestoneDB; -function MilestoneDB(options) { - emitter.EventEmitter.call(this); - - // The interval at which milestone snapshots should be saved - this.interval = options && options.interval; -} -emitter.mixin(MilestoneDB); - -MilestoneDB.prototype.close = function(callback) { - if (callback) util.nextTick(callback); -}; - -/** - * Fetch a milestone snapshot from the database - * @param {string} collection - name of the snapshot's collection - * @param {string} id - ID of the snapshot to fetch - * @param {number} version - the desired version of the milestone snapshot. The database will return - * the most recent milestone snapshot whose version is equal to or less than the provided value - * @param {Function} callback - a callback to invoke once the snapshot has been fetched. Should have - * the signature (error, snapshot) => void; - */ -MilestoneDB.prototype.getMilestoneSnapshot = function(collection, id, version, callback) { - var error = new ShareDBError( - ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, - 'getMilestoneSnapshot MilestoneDB method unimplemented' - ); - this._callBackOrEmitError(error, callback); -}; - -/** - * @param {string} collection - name of the snapshot's collection - * @param {Snapshot} snapshot - the milestone snapshot to save - * @param {Function} callback (optional) - a callback to invoke after the snapshot has been saved. - * Should have the signature (error) => void; - */ -MilestoneDB.prototype.saveMilestoneSnapshot = function(collection, snapshot, callback) { - var error = new ShareDBError( - ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, - 'saveMilestoneSnapshot MilestoneDB method unimplemented' - ); - this._callBackOrEmitError(error, callback); -}; - -MilestoneDB.prototype.getMilestoneSnapshotAtOrBeforeTime = function(collection, id, timestamp, callback) { - var error = new ShareDBError( - ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, - 'getMilestoneSnapshotAtOrBeforeTime MilestoneDB method unimplemented' - ); - this._callBackOrEmitError(error, callback); -}; - -MilestoneDB.prototype.getMilestoneSnapshotAtOrAfterTime = function(collection, id, timestamp, callback) { - var error = new ShareDBError( - ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, - 'getMilestoneSnapshotAtOrAfterTime MilestoneDB method unimplemented' - ); - this._callBackOrEmitError(error, callback); -}; - -MilestoneDB.prototype._isValidVersion = function(version) { - return util.isValidVersion(version); -}; - -MilestoneDB.prototype._isValidTimestamp = function(timestamp) { - return util.isValidTimestamp(timestamp); -}; - -MilestoneDB.prototype._callBackOrEmitError = function(error, callback) { - if (callback) return util.nextTick(callback, error); - this.emit('error', error); -}; diff --git a/lib/milestone-db/memory.js b/lib/milestone-db/memory.js deleted file mode 100644 index 00622363a..000000000 --- a/lib/milestone-db/memory.js +++ /dev/null @@ -1,141 +0,0 @@ -var MilestoneDB = require('./index'); -var ShareDBError = require('../error'); -var util = require('../util'); - -var ERROR_CODE = ShareDBError.CODES; - -/** - * In-memory ShareDB milestone database - * - * Milestone snapshots exist to speed up Backend.fetchSnapshot by providing milestones - * on top of which fewer ops can be applied to reach a desired version of the document. - * This very concept relies on persistence, which means that an in-memory database like - * this is in no way appropriate for production use. - * - * The main purpose of this class is to provide a simple example of implementation, - * and for use in tests. - */ -module.exports = MemoryMilestoneDB; -function MemoryMilestoneDB(options) { - MilestoneDB.call(this, options); - - // Map from collection name -> doc id -> array of milestone snapshots - this._milestoneSnapshots = Object.create(null); -} - -MemoryMilestoneDB.prototype = Object.create(MilestoneDB.prototype); - -MemoryMilestoneDB.prototype.getMilestoneSnapshot = function(collection, id, version, callback) { - if (!this._isValidVersion(version)) { - return util.nextTick(callback, new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Invalid version')); - } - - var predicate = versionLessThanOrEqualTo(version); - this._findMilestoneSnapshot(collection, id, predicate, callback); -}; - -MemoryMilestoneDB.prototype.saveMilestoneSnapshot = function(collection, snapshot, callback) { - callback = callback || function(error) { - if (error) return this.emit('error', error); - this.emit('save', collection, snapshot); - }.bind(this); - - if (!collection) return callback(new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Missing collection')); - if (!snapshot) return callback(new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Missing snapshot')); - - var milestoneSnapshots = this._getMilestoneSnapshotsSync(collection, snapshot.id); - milestoneSnapshots.push(snapshot); - milestoneSnapshots.sort(function(a, b) { - return a.v - b.v; - }); - - util.nextTick(callback, null); -}; - -MemoryMilestoneDB.prototype.getMilestoneSnapshotAtOrBeforeTime = function(collection, id, timestamp, callback) { - if (!this._isValidTimestamp(timestamp)) { - return util.nextTick(callback, new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Invalid timestamp')); - } - - var filter = timestampLessThanOrEqualTo(timestamp); - this._findMilestoneSnapshot(collection, id, filter, callback); -}; - -MemoryMilestoneDB.prototype.getMilestoneSnapshotAtOrAfterTime = function(collection, id, timestamp, callback) { - if (!this._isValidTimestamp(timestamp)) { - return util.nextTick(callback, new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Invalid timestamp')); - } - - var filter = timestampGreaterThanOrEqualTo(timestamp); - this._findMilestoneSnapshot(collection, id, filter, function(error, snapshot) { - if (error) return util.nextTick(callback, error); - - var mtime = snapshot && snapshot.m && snapshot.m.mtime; - if (timestamp !== null && mtime < timestamp) { - snapshot = undefined; - } - - util.nextTick(callback, null, snapshot); - }); -}; - -MemoryMilestoneDB.prototype._findMilestoneSnapshot = function(collection, id, breakCondition, callback) { - if (!collection) { - return util.nextTick( - callback, new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Missing collection') - ); - } - if (!id) return util.nextTick(callback, new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Missing ID')); - - var milestoneSnapshots = this._getMilestoneSnapshotsSync(collection, id); - - var milestoneSnapshot; - for (var i = 0; i < milestoneSnapshots.length; i++) { - var nextMilestoneSnapshot = milestoneSnapshots[i]; - if (breakCondition(milestoneSnapshot, nextMilestoneSnapshot)) { - break; - } else { - milestoneSnapshot = nextMilestoneSnapshot; - } - } - - util.nextTick(callback, null, milestoneSnapshot); -}; - -MemoryMilestoneDB.prototype._getMilestoneSnapshotsSync = function(collection, id) { - var collectionSnapshots = this._milestoneSnapshots[collection] || - (this._milestoneSnapshots[collection] = Object.create(null)); - return collectionSnapshots[id] || (collectionSnapshots[id] = []); -}; - -function versionLessThanOrEqualTo(version) { - return function(currentSnapshot, nextSnapshot) { - if (version === null) { - return false; - } - - return nextSnapshot.v > version; - }; -} - -function timestampGreaterThanOrEqualTo(timestamp) { - return function(currentSnapshot) { - if (timestamp === null) { - return false; - } - - var mtime = currentSnapshot && currentSnapshot.m && currentSnapshot.m.mtime; - return mtime >= timestamp; - }; -} - -function timestampLessThanOrEqualTo(timestamp) { - return function(currentSnapshot, nextSnapshot) { - if (timestamp === null) { - return !!currentSnapshot; - } - - var mtime = nextSnapshot && nextSnapshot.m && nextSnapshot.m.mtime; - return mtime > timestamp; - }; -} diff --git a/lib/milestone-db/no-op.js b/lib/milestone-db/no-op.js deleted file mode 100644 index f8b1f3822..000000000 --- a/lib/milestone-db/no-op.js +++ /dev/null @@ -1,35 +0,0 @@ -var MilestoneDB = require('./index'); -var util = require('../util'); - -/** - * A no-op implementation of the MilestoneDB class. - * - * This class exists as a simple, silent default drop-in for ShareDB, which allows the backend to call its methods with - * no effect. - */ -module.exports = NoOpMilestoneDB; -function NoOpMilestoneDB(options) { - MilestoneDB.call(this, options); -} - -NoOpMilestoneDB.prototype = Object.create(MilestoneDB.prototype); - -NoOpMilestoneDB.prototype.getMilestoneSnapshot = function(collection, id, version, callback) { - var snapshot = undefined; - util.nextTick(callback, null, snapshot); -}; - -NoOpMilestoneDB.prototype.saveMilestoneSnapshot = function(collection, snapshot, callback) { - if (callback) return util.nextTick(callback, null); - this.emit('save', collection, snapshot); -}; - -NoOpMilestoneDB.prototype.getMilestoneSnapshotAtOrBeforeTime = function(collection, id, timestamp, callback) { - var snapshot = undefined; - util.nextTick(callback, null, snapshot); -}; - -NoOpMilestoneDB.prototype.getMilestoneSnapshotAtOrAfterTime = function(collection, id, timestamp, callback) { - var snapshot = undefined; - util.nextTick(callback, null, snapshot); -}; diff --git a/lib/op-stream.js b/lib/op-stream.js deleted file mode 100644 index 6dd751a58..000000000 --- a/lib/op-stream.js +++ /dev/null @@ -1,35 +0,0 @@ -var Readable = require('stream').Readable; -var util = require('./util'); - -// Stream of operations. Subscribe returns one of these -function OpStream() { - Readable.call(this, {objectMode: true}); - this.id = null; - this.open = true; -} -module.exports = OpStream; - -util.inherits(OpStream, Readable); - -// This function is for notifying us that the stream is empty and needs data. -// For now, we'll just ignore the signal and assume the reader reads as fast -// as we fill it. I could add a buffer in this function, but really I don't -// think that is any better than the buffer implementation in nodejs streams -// themselves. -OpStream.prototype._read = util.doNothing; - -OpStream.prototype.pushData = function(data) { - // Ignore any messages after unsubscribe - if (!this.open) return; - // This data gets consumed in Agent#_subscribeToStream - this.push(data); -}; - -OpStream.prototype.destroy = function() { - // Only close stream once - if (!this.open) return; - this.open = false; - - this.push(null); - this.emit('close'); -}; diff --git a/lib/pubsub/index.js b/lib/pubsub/index.js deleted file mode 100644 index 5f8fad1f6..000000000 --- a/lib/pubsub/index.js +++ /dev/null @@ -1,138 +0,0 @@ -var emitter = require('../emitter'); -var OpStream = require('../op-stream'); -var ShareDBError = require('../error'); -var util = require('../util'); - -var ERROR_CODE = ShareDBError.CODES; - -function PubSub(options) { - if (!(this instanceof PubSub)) return new PubSub(options); - emitter.EventEmitter.call(this); - - this.prefix = options && options.prefix; - this.nextStreamId = 1; - this.streamsCount = 0; - // Maps channel -> id -> stream - this.streams = Object.create(null); - // State for tracking subscriptions. We track this.subscribed separately from - // the streams, since the stream gets added synchronously, and the subscribe - // isn't complete until the callback returns from Redis - // Maps channel -> true - this.subscribed = Object.create(null); - - var pubsub = this; - this._defaultCallback = function(err) { - if (err) return pubsub.emit('error', err); - }; -} -module.exports = PubSub; -emitter.mixin(PubSub); - -PubSub.prototype.close = function(callback) { - for (var channel in this.streams) { - var map = this.streams[channel]; - for (var id in map) { - map[id].destroy(); - } - } - if (callback) util.nextTick(callback); -}; - -PubSub.prototype._subscribe = function(channel, callback) { - util.nextTick(function() { - callback(new ShareDBError( - ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, - '_subscribe PubSub method unimplemented' - )); - }); -}; - -PubSub.prototype._unsubscribe = function(channel, callback) { - util.nextTick(function() { - callback(new ShareDBError( - ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, - '_unsubscribe PubSub method unimplemented' - )); - }); -}; - -PubSub.prototype._publish = function(channels, data, callback) { - util.nextTick(function() { - callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, '_publish PubSub method unimplemented')); - }); -}; - -PubSub.prototype.subscribe = function(channel, callback) { - if (!callback) callback = this._defaultCallback; - if (this.prefix) { - channel = this.prefix + ' ' + channel; - } - - var pubsub = this; - if (this.subscribed[channel]) { - util.nextTick(function() { - var stream = pubsub._createStream(channel); - callback(null, stream); - }); - return; - } - this._subscribe(channel, function(err) { - if (err) return callback(err); - pubsub.subscribed[channel] = true; - var stream = pubsub._createStream(channel); - callback(null, stream); - }); -}; - -PubSub.prototype.publish = function(channels, data, callback) { - if (!callback) callback = this._defaultCallback; - if (this.prefix) { - for (var i = 0; i < channels.length; i++) { - channels[i] = this.prefix + ' ' + channels[i]; - } - } - this._publish(channels, data, callback); -}; - -PubSub.prototype._emit = function(channel, data) { - var channelStreams = this.streams[channel]; - if (channelStreams) { - for (var id in channelStreams) { - channelStreams[id].pushData(data); - } - } -}; - -PubSub.prototype._createStream = function(channel) { - var stream = new OpStream(); - var pubsub = this; - stream.once('close', function() { - pubsub._removeStream(channel, stream); - }); - - this.streamsCount++; - var map = this.streams[channel] || (this.streams[channel] = Object.create(null)); - stream.id = this.nextStreamId++; - map[stream.id] = stream; - - return stream; -}; - -PubSub.prototype._removeStream = function(channel, stream) { - var map = this.streams[channel]; - if (!map) return; - - this.streamsCount--; - delete map[stream.id]; - - // Cleanup if this was the last subscribed stream for the channel - if (util.hasKeys(map)) return; - delete this.streams[channel]; - // Synchronously clear subscribed state. We won't actually be unsubscribed - // until some unknown time in the future. If subscribe is called in this - // period, we want to send a subscription message and wait for it to - // complete before we can count on being subscribed again - delete this.subscribed[channel]; - - this._unsubscribe(channel, this._defaultCallback); -}; diff --git a/lib/pubsub/memory.js b/lib/pubsub/memory.js deleted file mode 100644 index d01b533b2..000000000 --- a/lib/pubsub/memory.js +++ /dev/null @@ -1,39 +0,0 @@ -var PubSub = require('./index'); -var util = require('../util'); - -// In-memory ShareDB pub/sub -// -// This is a fully functional implementation. Since ShareDB does not require -// persistence of pub/sub state, it may be used in production environments -// requiring only a single stand alone server process. Additionally, it is -// easy to swap in an external pub/sub adapter if/when additional server -// processes are desired. No pub/sub APIs are adapter specific. - -function MemoryPubSub(options) { - if (!(this instanceof MemoryPubSub)) return new MemoryPubSub(options); - PubSub.call(this, options); -} -module.exports = MemoryPubSub; - -MemoryPubSub.prototype = Object.create(PubSub.prototype); - -MemoryPubSub.prototype._subscribe = function(channel, callback) { - util.nextTick(callback); -}; - -MemoryPubSub.prototype._unsubscribe = function(channel, callback) { - util.nextTick(callback); -}; - -MemoryPubSub.prototype._publish = function(channels, data, callback) { - var pubsub = this; - util.nextTick(function() { - for (var i = 0; i < channels.length; i++) { - var channel = channels[i]; - if (pubsub.subscribed[channel]) { - pubsub._emit(channel, data); - } - } - callback(); - }); -}; diff --git a/lib/query-emitter.js b/lib/query-emitter.js deleted file mode 100644 index cfbd930af..000000000 --- a/lib/query-emitter.js +++ /dev/null @@ -1,353 +0,0 @@ -var arraydiff = require('arraydiff'); -var deepEqual = require('fast-deep-equal'); -var ShareDBError = require('./error'); -var util = require('./util'); - -var ERROR_CODE = ShareDBError.CODES; - -function QueryEmitter(request, streams, ids, extra) { - this.backend = request.backend; - this.agent = request.agent; - this.db = request.db; - this.index = request.index; - this.query = request.query; - this.collection = request.collection; - this.fields = request.fields; - this.options = request.options; - this.snapshotProjection = request.snapshotProjection; - this.streams = streams; - this.ids = ids; - this.extra = extra; - - this.skipPoll = this.options.skipPoll || util.doNothing; - this.canPollDoc = this.db.canPollDoc(this.collection, this.query); - this.pollDebounce = - (typeof this.options.pollDebounce === 'number') ? this.options.pollDebounce : - (typeof this.db.pollDebounce === 'number') ? this.db.pollDebounce : - streams.length > 1 ? 1000 : 0; - this.pollInterval = - (typeof this.options.pollInterval === 'number') ? this.options.pollInterval : - (typeof this.db.pollInterval === 'number') ? this.db.pollInterval : - streams.length > 1 ? 1000 : 0; - - this._polling = false; - this._pendingPoll = null; - this._pollDebounceId = null; - this._pollIntervalId = null; -} -module.exports = QueryEmitter; - -// Start processing events from the stream -QueryEmitter.prototype._open = function() { - var emitter = this; - this._defaultCallback = function(err) { - if (err) emitter.onError(err); - }; - - emitter.streams.forEach(function(stream) { - stream.on('data', function(data) { - if (data.error) { - return emitter.onError(data.error); - } - emitter._update(data); - }); - stream.on('end', function() { - emitter.destroy(); - }); - }); - - // Make sure we start polling if pollInterval is being used - this._flushPoll(); -}; - -QueryEmitter.prototype.destroy = function() { - clearTimeout(this._pollDebounceId); - clearTimeout(this._pollIntervalId); - - var stream; - - while (stream = this.streams.pop()) { - stream.destroy(); - } -}; - -QueryEmitter.prototype._emitTiming = function(action, start) { - this.backend.emit('timing', action, Date.now() - start, this); -}; - -QueryEmitter.prototype._update = function(op) { - // Note that `op` should not be projected or sanitized yet. It's possible for - // a query to filter on a field that's not in the projection. skipPoll checks - // to see if an op could possibly affect a query, so it should get passed the - // full op. The onOp listener function must call backend.sanitizeOp() - var id = op.d; - var pollCallback = this._defaultCallback; - - // Check if the op's id matches the query before updating the query results - // and send it through immediately if it does. The current snapshot - // (including the op) for a newly matched document will get sent in the - // insert diff, so we don't need to send the op that caused the doc to - // match. If the doc already exists in the client and isn't otherwise - // subscribed, the client will need to request the op when it receives the - // snapshot from the query to bring itself up to date. - // - // The client may see the result of the op get reflected before the query - // results update. This might prove janky in some cases, since a doc could - // get deleted before it is removed from the results, for example. However, - // it will mean that ops which don't end up changing the results are - // received sooner even if query polling takes a while. - // - // Alternatively, we could send the op message only after the query has - // updated, and it would perhaps be ideal to send in the same message to - // avoid the user seeing transitional states where the doc is updated but - // the results order is not. - // - // We should send the op even if it is the op that causes the document to no - // longer match the query. If client-side filters are applied to the model - // to figure out which documents to render in a list, we will want the op - // that removed the doc from the query to cause the client-side computed - // list to update. - if (this.ids.indexOf(id) !== -1) { - var emitter = this; - pollCallback = function(err) { - // Send op regardless of polling error. Clients handle subscription to ops - // on the documents that currently match query results independently from - // updating which docs match the query - emitter.onOp(op); - if (err) emitter.onError(err); - }; - } - - // Ignore if the database or user function says we don't need to poll - try { - if ( - this.db.skipPoll(this.collection, id, op, this.query) || - this.skipPoll(this.collection, id, op, this.query) - ) { - return pollCallback(); - } - } catch (err) { - return pollCallback(err); - } - if (this.canPollDoc) { - // We can query against only the document that was modified to see if the - // op has changed whether or not it matches the results - this.queryPollDoc(id, pollCallback); - } else { - // We need to do a full poll of the query, because the query uses limits, - // sorts, or something special - this.queryPoll(pollCallback); - } -}; - -QueryEmitter.prototype._flushPoll = function() { - // Don't send another polling query at the same time or within the debounce - // timeout. This function will be called again once the poll that is - // currently in progress or the pollDebounce timeout completes - if (this._polling || this._pollDebounceId) return; - - // If another polling event happened while we were polling, call poll again, - // as the results may have changed - if (this._pendingPoll) { - this.queryPoll(); - - // If a pollInterval is specified, poll if the query doesn't get polled in - // the time of the interval - } else if (this.pollInterval) { - var emitter = this; - this._pollIntervalId = setTimeout(function() { - emitter._pollIntervalId = null; - emitter.queryPoll(emitter._defaultCallback); - }, this.pollInterval); - } -}; - -QueryEmitter.prototype.queryPoll = function(callback) { - var emitter = this; - - // Only run a single polling check against mongo at a time per emitter. This - // matters for two reasons: First, one callback could return before the - // other. Thus, our result diffs could get out of order, and the clients - // could end up with results in a funky order and the wrong results being - // mutated in the query. Second, only having one query executed - // simultaneously per emitter will act as a natural adaptive rate limiting - // in case the db is under load. - // - // This isn't necessary for the document polling case, since they operate - // on a given id and won't accidentally modify the wrong doc. Also, those - // queries should be faster and are less likely to be the same, so there is - // less benefit to possible load reduction. - if (this._polling || this._pollDebounceId) { - if (this._pendingPoll) { - this._pendingPoll.push(callback); - } else { - this._pendingPoll = [callback]; - } - return; - } - this._polling = true; - var pending = this._pendingPoll; - this._pendingPoll = null; - if (this.pollDebounce) { - this._pollDebounceId = setTimeout(function() { - emitter._pollDebounceId = null; - emitter._flushPoll(); - }, this.pollDebounce); - } - clearTimeout(this._pollIntervalId); - - var start = Date.now(); - this.db.queryPoll(this.collection, this.query, this.options, function(err, ids, extra) { - if (err) return emitter._finishPoll(err, callback, pending); - emitter._emitTiming('queryEmitter.poll', start); - - // Be nice to not have to do this in such a brute force way - if (!deepEqual(emitter.extra, extra)) { - emitter.extra = extra; - emitter.onExtra(extra); - } - - var idsDiff = arraydiff(emitter.ids, ids); - if (idsDiff.length) { - emitter.ids = ids; - var inserted = getInserted(idsDiff); - if (inserted.length) { - var snapshotOptions = {}; - snapshotOptions.agentCustom = emitter.agent.custom; - - function _getSnapshotBulkCb(err, snapshotMap) { - if (err) return emitter._finishPoll(err, callback, pending); - var snapshots = emitter.backend._getSnapshotsFromMap(inserted, snapshotMap); - var snapshotType = emitter.backend.SNAPSHOT_TYPES.current; - emitter.backend._sanitizeSnapshots( - emitter.agent, - emitter.snapshotProjection, - emitter.collection, - snapshots, - snapshotType, - function(err) { - if (err) return emitter._finishPoll(err, callback, pending); - emitter._emitTiming('queryEmitter.pollGetSnapshotBulk', start); - var diff = mapDiff(idsDiff, snapshotMap); - emitter.onDiff(diff); - emitter._finishPoll(err, callback, pending); - }); - }; - emitter.db.getSnapshotBulk(emitter.collection, inserted, emitter.fields, snapshotOptions, _getSnapshotBulkCb); - } else { - emitter.onDiff(idsDiff); - emitter._finishPoll(err, callback, pending); - } - } else { - emitter._finishPoll(err, callback, pending); - } - }); -}; - -QueryEmitter.prototype._finishPoll = function(err, callback, pending) { - this._polling = false; - if (callback) callback(err); - if (pending) { - for (var i = 0; i < pending.length; i++) { - callback = pending[i]; - if (callback) callback(err); - } - } - this._flushPoll(); -}; - -QueryEmitter.prototype.queryPollDoc = function(id, callback) { - var emitter = this; - var start = Date.now(); - this.db.queryPollDoc(this.collection, id, this.query, this.options, function(err, matches) { - if (err) return callback(err); - emitter._emitTiming('queryEmitter.pollDoc', start); - - // Check if the document was in the previous results set - var i = emitter.ids.indexOf(id); - - if (i === -1 && matches) { - // Add doc to the collection. Order isn't important, so we'll just whack - // it at the end - var index = emitter.ids.push(id) - 1; - - var snapshotOptions = {}; - snapshotOptions.agentCustom = emitter.agent.custom; - - // We can get the result to send to the client async, since there is a - // delay in sending to the client anyway - emitter.db.getSnapshot(emitter.collection, id, emitter.fields, snapshotOptions, function(err, snapshot) { - if (err) return callback(err); - var snapshots = [snapshot]; - var snapshotType = emitter.backend.SNAPSHOT_TYPES.current; - emitter.backend._sanitizeSnapshots( - emitter.agent, - emitter.snapshotProjection, - emitter.collection, - snapshots, - snapshotType, - function(err) { - if (err) return callback(err); - emitter.onDiff([new arraydiff.InsertDiff(index, snapshots)]); - emitter._emitTiming('queryEmitter.pollDocGetSnapshot', start); - callback(); - }); - }); - return; - } - - if (i !== -1 && !matches) { - emitter.ids.splice(i, 1); - emitter.onDiff([new arraydiff.RemoveDiff(i, 1)]); - return callback(); - } - - callback(); - }); -}; - -// Clients must assign each of these functions synchronously after constructing -// an instance of QueryEmitter. The instance is subscribed to an op stream at -// construction time, and does not buffer emitted events. Diff events assume -// all messages are received and applied in order, so it is critical that none -// are dropped. -QueryEmitter.prototype.onError = - QueryEmitter.prototype.onDiff = - QueryEmitter.prototype.onExtra = - QueryEmitter.prototype.onOp = function() { - throw new ShareDBError( - ERROR_CODE.ERR_QUERY_EMITTER_LISTENER_NOT_ASSIGNED, - 'Required QueryEmitter listener not assigned' - ); - }; - -function getInserted(diff) { - var inserted = []; - for (var i = 0; i < diff.length; i++) { - var item = diff[i]; - if (item instanceof arraydiff.InsertDiff) { - for (var j = 0; j < item.values.length; j++) { - inserted.push(item.values[j]); - } - } - } - return inserted; -} - -function mapDiff(idsDiff, snapshotMap) { - var diff = []; - for (var i = 0; i < idsDiff.length; i++) { - var item = idsDiff[i]; - if (item instanceof arraydiff.InsertDiff) { - var values = []; - for (var j = 0; j < item.values.length; j++) { - var id = item.values[j]; - values.push(snapshotMap[id]); - } - diff.push(new arraydiff.InsertDiff(item.index, values)); - } else { - diff.push(item); - } - } - return diff; -} diff --git a/lib/read-snapshots-request.js b/lib/read-snapshots-request.js deleted file mode 100644 index 8a595698f..000000000 --- a/lib/read-snapshots-request.js +++ /dev/null @@ -1,109 +0,0 @@ -var ShareDBError = require('./error'); - -module.exports = ReadSnapshotsRequest; - -/** - * Context object passed to "readSnapshots" middleware functions - * - * @param {string} collection - * @param {Snapshot[]} snapshots - snapshots being read - * @param {keyof Backend.prototype.SNAPSHOT_TYPES} snapshotType - the type of snapshot read being - * performed - */ -function ReadSnapshotsRequest(collection, snapshots, snapshotType) { - this.collection = collection; - this.snapshots = snapshots; - this.snapshotType = snapshotType; - - // Added by Backend#trigger - this.action = null; - this.agent = null; - this.backend = null; - - /** - * Map of doc id to error: `{[docId: string]: string | Error}` - */ - this._idToError = null; -} - -/** - * Rejects the read of a specific snapshot. A rejected snapshot read will not have that snapshot's - * data sent down to the client. - * - * If the error has a `code` property of `"ERR_SNAPSHOT_READ_SILENT_REJECTION"`, then the Share - * client will not pass the error to user code, but will still do things like cancel subscriptions. - * The `#rejectSnapshotReadSilent(snapshot, errorMessage)` method can also be used for convenience. - * - * @param {Snapshot} snapshot - * @param {string | Error} error - * - * @see #rejectSnapshotReadSilent - * @see ShareDBError.CODES.ERR_SNAPSHOT_READ_SILENT_REJECTION - * @see ShareDBError.CODES.ERR_SNAPSHOT_READS_REJECTED - */ -ReadSnapshotsRequest.prototype.rejectSnapshotRead = function(snapshot, error) { - if (!this._idToError) { - this._idToError = Object.create(null); - } - this._idToError[snapshot.id] = error; -}; - -/** - * Rejects the read of a specific snapshot. A rejected snapshot read will not have that snapshot's - * data sent down to the client. - * - * This method will set a special error code that causes the Share client to not pass the error to - * user code, though it will still do things like cancel subscriptions. - * - * @param {Snapshot} snapshot - * @param {string} errorMessage - */ -ReadSnapshotsRequest.prototype.rejectSnapshotReadSilent = function(snapshot, errorMessage) { - this.rejectSnapshotRead(snapshot, this.silentRejectionError(errorMessage)); -}; - -ReadSnapshotsRequest.prototype.silentRejectionError = function(errorMessage) { - return new ShareDBError(ShareDBError.CODES.ERR_SNAPSHOT_READ_SILENT_REJECTION, errorMessage); -}; - -/** - * Returns whether this trigger of "readSnapshots" has had a snapshot read rejected. - */ -ReadSnapshotsRequest.prototype.hasSnapshotRejection = function() { - return this._idToError != null; -}; - -/** - * Returns an overall error from "readSnapshots" based on the snapshot-specific errors. - * - * - If there's exactly one snapshot and it has an error, then that error is returned. - * - If there's more than one snapshot and at least one has an error, then an overall - * "ERR_SNAPSHOT_READS_REJECTED" is returned, with an `idToError` property. - */ -ReadSnapshotsRequest.prototype.getReadSnapshotsError = function() { - var snapshots = this.snapshots; - var idToError = this._idToError; - // If there are 0 snapshots, there can't be any snapshot-specific errors. - if (snapshots.length === 0) { - return; - } - - // Single snapshot with error is treated as a full error. - if (snapshots.length === 1) { - var snapshotError = idToError[snapshots[0].id]; - if (snapshotError) { - return snapshotError; - } else { - return; - } - } - - // Errors in specific snapshots result in an overall ERR_SNAPSHOT_READS_REJECTED. - // - // fetchBulk and subscribeBulk know how to handle that special error by sending a doc-by-doc - // success/failure to the client. Other methods that don't or can't handle partial failures - // will treat it as a full rejection. - var err = new ShareDBError(ShareDBError.CODES.ERR_SNAPSHOT_READS_REJECTED); - err.idToError = idToError; - return err; -}; diff --git a/lib/snapshot.js b/lib/snapshot.js deleted file mode 100644 index 548a7e25b..000000000 --- a/lib/snapshot.js +++ /dev/null @@ -1,8 +0,0 @@ -module.exports = Snapshot; -function Snapshot(id, version, type, data, meta) { - this.id = id; - this.v = version; - this.type = type; - this.data = data; - this.m = meta; -} diff --git a/lib/stream-socket.js b/lib/stream-socket.js deleted file mode 100644 index fa9b6f98b..000000000 --- a/lib/stream-socket.js +++ /dev/null @@ -1,62 +0,0 @@ -var Duplex = require('stream').Duplex; -var logger = require('./logger'); -var util = require('./util'); - -function StreamSocket() { - this.readyState = 0; - this.stream = new ServerStream(this); -} -module.exports = StreamSocket; - -StreamSocket.prototype._open = function() { - if (this.readyState !== 0) return; - this.readyState = 1; - this.onopen(); -}; -StreamSocket.prototype.close = function(reason) { - if (this.readyState === 3) return; - this.readyState = 3; - // Signal data writing is complete. Emits the 'end' event - this.stream.push(null); - this.onclose(reason || 'closed'); -}; -StreamSocket.prototype.send = function(data) { - // Data is an object - this.stream.push(JSON.parse(data)); -}; -StreamSocket.prototype.onmessage = util.doNothing; -StreamSocket.prototype.onclose = util.doNothing; -StreamSocket.prototype.onerror = util.doNothing; -StreamSocket.prototype.onopen = util.doNothing; - - -function ServerStream(socket) { - Duplex.call(this, {objectMode: true}); - - this.socket = socket; - - this.on('error', function(error) { - logger.warn('ShareDB client message stream error', error); - socket.close('stopped'); - }); - - // The server ended the writable stream. Triggered by calling stream.end() - // in agent.close() - this.on('finish', function() { - socket.close('stopped'); - }); -} -util.inherits(ServerStream, Duplex); - -ServerStream.prototype.isServer = true; - -ServerStream.prototype._read = util.doNothing; - -ServerStream.prototype._write = function(chunk, encoding, callback) { - var socket = this.socket; - util.nextTick(function() { - if (socket.readyState !== 1) return; - socket.onmessage({data: JSON.stringify(chunk)}); - callback(); - }); -}; diff --git a/lib/submit-request.js b/lib/submit-request.js deleted file mode 100644 index 8e6715123..000000000 --- a/lib/submit-request.js +++ /dev/null @@ -1,382 +0,0 @@ -var ot = require('./ot'); -var projections = require('./projections'); -var ShareDBError = require('./error'); -var types = require('./types'); -var protocol = require('./protocol'); - -var ERROR_CODE = ShareDBError.CODES; - -function SubmitRequest(backend, agent, index, id, op, options) { - this.backend = backend; - this.agent = agent; - // If a projection, rewrite the call into a call against the collection - var projection = backend.projections[index]; - this.index = index; - this.projection = projection; - this.collection = (projection) ? projection.target : index; - this.id = id; - this.op = op; - this.options = options; - - this.extra = op.x; - delete op.x; - - this.start = Date.now(); - this._addOpMeta(); - - // Set as this request is sent through middleware - this.action = null; - // For custom use in middleware - this.custom = Object.create(null); - - // Whether or not to store a milestone snapshot. If left as null, the milestone - // snapshots are saved according to the interval provided to the milestone db - // options. If overridden to a boolean value, then that value is used instead of - // the interval logic. - this.saveMilestoneSnapshot = null; - this.suppressPublish = backend.suppressPublish; - this.maxRetries = backend.maxSubmitRetries; - this.retries = 0; - - // return values - this.snapshot = null; - this.ops = []; - this.channels = null; - this._fixupOps = []; -} -module.exports = SubmitRequest; - -SubmitRequest.prototype.$fixup = function(op) { - if (this.action !== this.backend.MIDDLEWARE_ACTIONS.apply) { - throw new ShareDBError( - ERROR_CODE.ERR_FIXUP_IS_ONLY_VALID_ON_APPLY, - 'fixup can only be called during the apply middleware' - ); - } - - if (this.op.del) { - throw new ShareDBError( - ERROR_CODE.ERR_CANNOT_FIXUP_DELETION, - 'fixup cannot be applied on deletion ops' - ); - } - - var typeId = this.op.create ? this.op.create.type : this.snapshot.type; - var type = types.map[typeId]; - if (typeof type.compose !== 'function') { - throw new ShareDBError( - ERROR_CODE.ERR_TYPE_DOES_NOT_SUPPORT_COMPOSE, - typeId + ' does not support compose' - ); - } - - if (this.op.create) this.op.create.data = type.apply(this.op.create.data, op); - else this.op.op = type.compose(this.op.op, op); - - var fixupOp = { - src: this.op.src, - seq: this.op.seq, - v: this.op.v, - op: op - }; - - this._fixupOps.push(fixupOp); -}; - -SubmitRequest.prototype.submit = function(callback) { - var request = this; - var backend = this.backend; - var collection = this.collection; - var id = this.id; - var op = this.op; - // Send a special projection so that getSnapshot knows to return all fields. - // With a null projection, it strips document metadata - var fields = {$submit: true}; - - var snapshotOptions = {}; - snapshotOptions.agentCustom = request.agent.custom; - backend.db.getSnapshot(collection, id, fields, snapshotOptions, function(err, snapshot) { - if (err) return callback(err); - - request.snapshot = snapshot; - request._addSnapshotMeta(); - - if (op.v == null) { - if (op.create && snapshot.type && op.src) { - // If the document was already created by another op, we will return a - // 'Document already exists' error in response and fail to submit this - // op. However, this could also happen in the case that the op was - // already committed and the create op was simply resent. In that - // case, we should return a non-fatal 'Op already submitted' error. We - // must get the past ops and check their src and seq values to - // differentiate. - request._fetchCreateOpVersion(function(error, version) { - if (error) return callback(error); - if (version == null) { - callback(request.alreadyCreatedError()); - } else { - op.v = version; - callback(request.alreadySubmittedError()); - } - }); - return; - } - - // Submitting an op with a null version means that it should get the - // version from the latest snapshot. Generally this will mean the op - // won't be transformed, though transform could be called on it in the - // case of a retry from a simultaneous submit - op.v = snapshot.v; - } - - if (op.v === snapshot.v) { - // The snapshot hasn't changed since the op's base version. Apply - // without transforming the op - return request.apply(callback); - } - - if (op.v > snapshot.v) { - // The op version should be from a previous snapshot, so it should never - // never exceed the current snapshot's version - return callback(request.newerVersionError()); - } - - // Transform the op up to the current snapshot version, then apply - var from = op.v; - backend.db.getOpsToSnapshot(collection, id, from, snapshot, {metadata: true}, function(err, ops) { - if (err) return callback(err); - - if (ops.length !== snapshot.v - from) { - return callback(request.missingOpsError()); - } - - err = request._transformOp(ops); - if (err) return callback(err); - - var skipNoOp = backend.doNotCommitNoOps && - protocol.checkAtLeast(request.agent.protocol, '1.2') && - request.op.op && - request.op.op.length === 0; - - if (skipNoOp) { - // The op is a no-op, either because it was submitted as such, or - more - // likely - because it was transformed into one. Let's avoid committing it - // and tell the client. - return callback(request.noOpError()); - } - - if (op.v !== snapshot.v) { - // This shouldn't happen, but is just a final sanity check to make - // sure we have transformed the op to the current snapshot version - return callback(request.versionAfterTransformError()); - } - - request.apply(callback); - }); - }); -}; - -SubmitRequest.prototype.apply = function(callback) { - // If we're being projected, verify that the op is allowed - var projection = this.projection; - if (projection && !projections.isOpAllowed(this.snapshot.type, projection.fields, this.op)) { - return callback(this.projectionError()); - } - - // Always set the channels before each attempt to apply. If the channels are - // modified in a middleware and we retry, we want to reset to a new array - this.channels = this.backend.getChannels(this.collection, this.id); - this._fixupOps = []; - delete this.op.m.fixup; - - var request = this; - this.backend.trigger(this.backend.MIDDLEWARE_ACTIONS.apply, this.agent, this, function(err) { - if (err) return callback(err); - - // Apply the submitted op to the snapshot - err = ot.apply(request.snapshot, request.op); - if (err) return callback(err); - - request.commit(callback); - }); -}; - -SubmitRequest.prototype.commit = function(callback) { - var request = this; - var backend = this.backend; - backend.trigger(backend.MIDDLEWARE_ACTIONS.commit, this.agent, this, function(err) { - if (err) return callback(err); - if (request._fixupOps.length) request.op.m.fixup = request._fixupOps; - if (request.op.create) { - // When we create the snapshot, we store a pointer to the op that created - // it. This allows us to return OP_ALREADY_SUBMITTED errors when appropriate. - request.snapshot.m._create = { - src: request.op.src, - seq: request.op.seq, - v: request.op.v - }; - } - - // Try committing the operation and snapshot to the database atomically - backend.db.commit( - request.collection, - request.id, - request.op, - request.snapshot, - request.options, - function(err, succeeded) { - if (err) return callback(err); - if (!succeeded) { - // Between our fetch and our call to commit, another client committed an - // operation. We expect this to be relatively infrequent but normal. - return request.retry(callback); - } - if (!request.suppressPublish) { - var op = request.op; - op.c = request.collection; - op.d = request.id; - op.m = undefined; - // Needed for agent to detect if it can ignore sending the op back to - // the client that submitted it in subscriptions - if (request.collection !== request.index) op.i = request.index; - backend.pubsub.publish(request.channels, op); - } - if (request._shouldSaveMilestoneSnapshot(request.snapshot)) { - request.backend.milestoneDb.saveMilestoneSnapshot(request.collection, request.snapshot); - } - callback(); - }); - }); -}; - -SubmitRequest.prototype.retry = function(callback) { - this.retries++; - if (this.maxRetries != null && this.retries > this.maxRetries) { - return callback(this.maxRetriesError()); - } - this.backend.emit('timing', 'submit.retry', Date.now() - this.start, this); - this.submit(callback); -}; - -SubmitRequest.prototype._transformOp = function(ops) { - var type = this.snapshot.type; - for (var i = 0; i < ops.length; i++) { - var op = ops[i]; - - if (this.op.src && this.op.src === op.src && this.op.seq === op.seq) { - // The op has already been submitted. There are a variety of ways this - // can happen in normal operation, such as a client resending an - // unacknowledged operation at reconnect. It's important we don't apply - // the same op twice - if (op.m.fixup) this._fixupOps = op.m.fixup; - return this.alreadySubmittedError(); - } - - if (this.op.v !== op.v) { - return this.versionDuringTransformError(); - } - - var err = ot.transform(type, this.op, op); - if (err) return err; - delete op.m; - this.ops.push(op); - } -}; - -SubmitRequest.prototype._addOpMeta = function() { - this.op.m = { - ts: this.start - }; - if (this.op.create) { - // Consistently store the full URI of the type, not just its short name - this.op.create.type = ot.normalizeType(this.op.create.type); - } -}; - -SubmitRequest.prototype._addSnapshotMeta = function() { - var meta = this.snapshot.m || (this.snapshot.m = {}); - if (this.op.create) { - meta.ctime = this.start; - } else if (this.op.del) { - this.op.m.data = this.snapshot.data; - } - meta.mtime = this.start; -}; - -SubmitRequest.prototype._shouldSaveMilestoneSnapshot = function(snapshot) { - // If the flag is null, it's not been overridden by the consumer, so apply the interval - if (this.saveMilestoneSnapshot === null) { - return snapshot && snapshot.v % this.backend.milestoneDb.interval === 0; - } - - return this.saveMilestoneSnapshot; -}; - -SubmitRequest.prototype._fetchCreateOpVersion = function(callback) { - var create = this.snapshot.m._create; - if (create) { - var version = (create.src === this.op.src && create.seq === this.op.seq) ? create.v : null; - return callback(null, version); - } - - // We can only reach here if the snapshot is missing the create metadata. - // This can happen if a client tries to re-create or resubmit a create op to - // a "legacy" snapshot that existed before we started adding the meta (should - // be uncommon) or when using a driver that doesn't support metadata (eg Postgres). - this.backend.db.getCommittedOpVersion(this.collection, this.id, this.snapshot, this.op, null, callback); -}; - -// Non-fatal client errors: -SubmitRequest.prototype.alreadySubmittedError = function() { - return new ShareDBError(ERROR_CODE.ERR_OP_ALREADY_SUBMITTED, 'Op already submitted'); -}; -SubmitRequest.prototype.rejectedError = function() { - return new ShareDBError(ERROR_CODE.ERR_OP_SUBMIT_REJECTED, 'Op submit rejected'); -}; -// Fatal client errors: -SubmitRequest.prototype.alreadyCreatedError = function() { - return new ShareDBError(ERROR_CODE.ERR_DOC_ALREADY_CREATED, 'Invalid op submitted. Document already created'); -}; -SubmitRequest.prototype.newerVersionError = function() { - return new ShareDBError( - ERROR_CODE.ERR_OP_VERSION_NEWER_THAN_CURRENT_SNAPSHOT, - 'Invalid op submitted. Op version newer than current snapshot' - ); -}; -SubmitRequest.prototype.projectionError = function() { - return new ShareDBError( - ERROR_CODE.ERR_OP_NOT_ALLOWED_IN_PROJECTION, - 'Invalid op submitted. Operation invalid in projected collection' - ); -}; -// Fatal internal errors: -SubmitRequest.prototype.missingOpsError = function() { - return new ShareDBError( - ERROR_CODE.ERR_SUBMIT_TRANSFORM_OPS_NOT_FOUND, - 'Op submit failed. DB missing ops needed to transform it up to the current snapshot version' - ); -}; -SubmitRequest.prototype.versionDuringTransformError = function() { - return new ShareDBError( - ERROR_CODE.ERR_OP_VERSION_MISMATCH_DURING_TRANSFORM, - 'Op submit failed. Versions mismatched during op transform' - ); -}; -SubmitRequest.prototype.versionAfterTransformError = function() { - return new ShareDBError( - ERROR_CODE.ERR_OP_VERSION_MISMATCH_AFTER_TRANSFORM, - 'Op submit failed. Op version mismatches snapshot after op transform' - ); -}; -SubmitRequest.prototype.maxRetriesError = function() { - return new ShareDBError( - ERROR_CODE.ERR_MAX_SUBMIT_RETRIES_EXCEEDED, - 'Op submit failed. Exceeded max submit retries of ' + this.maxRetries - ); -}; -SubmitRequest.prototype.noOpError = function() { - return new ShareDBError( - ERROR_CODE.ERR_NO_OP, - 'Op is a no-op. Skipping apply.' - ); -}; diff --git a/lib/types.js b/lib/types.js deleted file mode 100644 index f966ab9fe..000000000 --- a/lib/types.js +++ /dev/null @@ -1,11 +0,0 @@ - -exports.defaultType = require('ot-json0').type; - -exports.map = Object.create(null); - -exports.register = function(type) { - if (type.name) exports.map[type.name] = type; - if (type.uri) exports.map[type.uri] = type; -}; - -exports.register(exports.defaultType); diff --git a/package.json b/package.json index dd93b9422..aad742d4e 100644 --- a/package.json +++ b/package.json @@ -11,18 +11,21 @@ "ot-json0": "^1.1.0" }, "devDependencies": { - "chai": "^4.3.7", - "coveralls": "^3.1.1", - "eslint": "^8.47.0", + "@eslint/js": "^10.0.1", + "@types/node": "^24.12.0", + "chai": "^6.2.2", + "eslint": "^10.1.0", "eslint-config-google": "^0.14.0", - "mocha": "^10.2.0", - "nyc": "^15.1.0", + "mocha": "^11.7.5", + "nyc": "^18.0.0", "ot-json0-v2": "https://github.com/ottypes/json0#90a3ae26364c4fa3b19b6df34dad46707a704421", "ot-json1": "^1.0.2", "rich-text": "^4.1.0", "sharedb-legacy": "npm:sharedb@1.1.0", - "sinon": "^15.2.0", - "sinon-chai": "^3.7.0" + "sinon": "^21.0.3", + "sinon-chai": "^4.0.1", + "typescript": "~6.0.2", + "typescript-eslint": "^8.58.0" }, "files": [ "lib/", @@ -32,10 +35,11 @@ "docs:install": "cd docs && bundle install", "docs:build": "cd docs && bundle exec jekyll build", "docs:start": "cd docs && bundle exec jekyll serve --livereload", + "pretest": "tsc || echo '(ignoring TS errors for now)'", "test": "mocha", "test-cover": "nyc --temp-dir=coverage -r text -r lcov npm test", - "lint": "./node_modules/.bin/eslint --ignore-path .gitignore '**/*.js'", - "lint:fix": "npm run lint -- --fix" + "lint": "eslint", + "lint:fix": "eslint --fix" }, "repository": { "type": "git", diff --git a/src/agent.ts b/src/agent.ts new file mode 100644 index 000000000..a2fe3a0dd --- /dev/null +++ b/src/agent.ts @@ -0,0 +1,1094 @@ +import hat = require('hat'); +import ShareDBError = require('./error'); +import logger = require('./logger'); +import { ACTIONS } from './message-actions'; +import types = require('./types'); +import util = require('./util'); +import protocol = require('./protocol'); + +var ERROR_CODE = ShareDBError.CODES; + +/** + * Agent deserializes the wire protocol messages received from the stream and + * calls the corresponding functions on its Agent. It uses the return values + * to send responses back. Agent also handles piping the operation streams + * provided by a Agent. + * + * @param {Backend} backend + * @param {Duplex} stream connection to a client + */ +class Agent { + backend; + stream; + clientId; + src; + connectTime; + subscribedDocs; + subscribedQueries; + subscribedPresences; + presenceSubscriptionSeq; + presenceRequests; + latestDocVersionStreams; + latestDocVersions; + closed; + custom; + protocol; + _firstReceivedMessage; + _handshakeReceived; + + constructor(backend, stream) { + this.backend = backend; + this.stream = stream; + + this.clientId = hat(); + // src is a client-configurable "id" which the client will set in its handshake, + // and attach to its ops. This should take precedence over clientId if set. + // Only legacy clients, or new clients connecting for the first time will use the + // Agent-provided clientId. Ideally we'll deprecate clientId in favour of src + // in the next breaking change. + this.src = null; + this.connectTime = Date.now(); + + // We need to track which documents are subscribed by the client. This is a + // map of collection -> id -> stream + this.subscribedDocs = Object.create(null); + + // Map from queryId -> emitter + this.subscribedQueries = Object.create(null); + + // Track which documents are subscribed to presence by the client. This is a + // map of channel -> stream + this.subscribedPresences = Object.create(null); + // Highest seq received for a subscription request. Any seq lower than this + // value is stale, and should be ignored. Used for keeping the subscription + // state in sync with the client's desired state. Map of channel -> seq + this.presenceSubscriptionSeq = Object.create(null); + // Keep track of the last request that has been sent by each local presence + // belonging to this agent. This is used to generate a new disconnection + // request if the client disconnects ungracefully. This is a + // map of channel -> id -> request + this.presenceRequests = Object.create(null); + // Keep track of the latest known Doc version, so that we can avoid fetching + // ops to transform presence if not needed + this.latestDocVersionStreams = Object.create(null); + this.latestDocVersions = Object.create(null); + + // We need to track this manually to make sure we don't reply to messages + // after the stream was closed. + this.closed = false; + + // For custom use in middleware. The agent is a convenient place to cache + // session state in memory. It is in memory only as long as the session is + // active, and it is passed to each middleware call + this.custom = Object.create(null); + + this.protocol = Object.create(null); + + // The first message received over the connection. Stored to warn if messages + // are being sent before the handshake. + this._firstReceivedMessage = null; + this._handshakeReceived = false; + + // Send the legacy message to initialize old clients with the random agent Id + this.send(this._initMessage(ACTIONS.initLegacy)); + } + + // Close the agent with the client. + close(err) { + if (err) { + logger.warn('Agent closed due to error', this._src(), err.stack || err); + } + if (this.closed) return; + // This will end the writable stream and emit 'finish' + this.stream.end(); + } + + _cleanup() { + // Only clean up once if the stream emits both 'end' and 'close'. + if (this.closed) return; + + this.closed = true; + + this.backend.agentsCount--; + if (!this.stream.isServer) this.backend.remoteAgentsCount--; + + // Clean up doc subscription streams + for (var collection in this.subscribedDocs) { + var docs = this.subscribedDocs[collection]; + for (var id in docs) { + var stream = docs[id]; + stream.destroy(); + } + } + this.subscribedDocs = Object.create(null); + + for (var channel in this.subscribedPresences) { + this.subscribedPresences[channel].destroy(); + } + this.subscribedPresences = Object.create(null); + + // Clean up query subscription streams + for (var id in this.subscribedQueries) { + var emitter = this.subscribedQueries[id]; + emitter.destroy(); + } + this.subscribedQueries = Object.create(null); + + for (var collection in this.latestDocVersionStreams) { + var streams = this.latestDocVersionStreams[collection]; + for (var id in streams) streams[id].destroy(); + } + this.latestDocVersionStreams = Object.create(null); + } + + /** + * Passes operation data received on stream to the agent stream via + * _sendOp() + */ + _subscribeToStream(collection, id, stream) { + var agent = this; + this._subscribeMapToStream(this.subscribedDocs, collection, id, stream, function(data) { + if (data.error) { + // Log then silently ignore errors in a subscription stream, since these + // may not be the client's fault, and they were not the result of a + // direct request by the client + logger.error('Doc subscription stream error', collection, id, data.error); + return; + } + agent._onOp(collection, id, data); + }); + } + + _subscribeMapToStream(map, collection, id, stream, dataHandler) { + if (this.closed) return stream.destroy(); + + var streams = map[collection] || (map[collection] = Object.create(null)); + + // If already subscribed to this document, destroy the previously subscribed stream + var previous = streams[id]; + if (previous) previous.destroy(); + streams[id] = stream; + + stream.on('data', dataHandler); + stream.on('end', function() { + // The op stream is done sending, so release its reference + var streams = map[collection]; + if (!streams || streams[id] !== stream) return; + delete streams[id]; + if (util.hasKeys(streams)) return; + delete map[collection]; + }); + } + + _subscribeToPresenceStream(channel, stream) { + if (this.closed) return stream.destroy(); + var agent = this; + + stream.on('data', function(data) { + if (data.error) { + logger.error('Presence subscription stream error', channel, data.error); + } + agent._handlePresenceData(data); + }); + + stream.on('end', function() { + var requests = agent.presenceRequests[channel] || {}; + for (var id in requests) { + var request = agent.presenceRequests[channel][id]; + request.seq++; + request.p = null; + agent._broadcastPresence(request, function(error) { + if (error) logger.error('Error broadcasting disconnect presence', channel, error); + }); + } + if (agent.subscribedPresences[channel] === stream) { + delete agent.subscribedPresences[channel]; + } + delete agent.presenceRequests[channel]; + }); + } + + _subscribeToQuery(emitter, queryId, collection, query) { + var previous = this.subscribedQueries[queryId]; + if (previous) previous.destroy(); + this.subscribedQueries[queryId] = emitter; + + var agent = this; + emitter.onExtra = function(extra) { + agent.send({a: ACTIONS.queryUpdate, id: queryId, extra: extra}); + }; + + emitter.onDiff = function(diff) { + for (var i = 0; i < diff.length; i++) { + var item = diff[i]; + if (item.type === 'insert') { + item.values = getResultsData(item.values); + } + } + // Consider stripping the collection out of the data we send here + // if it matches the query's collection. + agent.send({a: ACTIONS.queryUpdate, id: queryId, diff: diff}); + }; + + emitter.onError = function(err) { + // Log then silently ignore errors in a subscription stream, since these + // may not be the client's fault, and they were not the result of a + // direct request by the client + logger.error('Query subscription stream error', collection, query, err); + }; + + emitter.onOp = function(op) { + var id = op.d; + agent._onOp(collection, id, op); + }; + + emitter._open(); + } + + _onOp(collection, id, op) { + if (this._isOwnOp(collection, op)) return; + + // Ops emitted here are coming directly from pubsub, which emits the same op + // object to listeners without making a copy. The pattern in middleware is to + // manipulate the passed in object, and projections are implemented the same + // way currently. + // + // Deep copying the op would be safest, but deep copies are very expensive, + // especially over arbitrary objects. This function makes a shallow copy of an + // op, and it requires that projections and any user middleware copy deep + // properties as needed when they modify the op. + // + // Polling of query subscriptions is determined by the same op objects. As a + // precaution against op middleware breaking query subscriptions, we delay + // before calling into projection and middleware code + var agent = this; + util.nextTick(function() { + var copy = shallowCopy(op); + agent.backend.sanitizeOp(agent, collection, id, copy, function(err) { + if (err) { + logger.error('Error sanitizing op emitted from subscription', collection, id, copy, err); + return; + } + agent._sendOp(collection, id, copy); + }); + }); + } + + _isOwnOp(collection, op) { + // Detect ops from this client on the same projection. Since the client sent + // these in, the submit reply will be sufficient and we can silently ignore + // them in the streams for subscribed documents or queries + return (this._src() === op.src) && (collection === (op.i || op.c)); + } + + send(message) { + // Quietly drop replies if the stream was closed + if (this.closed) return; + + this.backend.emit('send', this, message); + this.stream.write(message); + } + + _sendOp(collection, id, op) { + var message = { + a: ACTIONS.op, + c: collection, + d: id, + v: op.v, + src: op.src, + seq: op.seq + }; + if ('op' in op) message.op = op.op; + if (op.create) message.create = op.create; + if (op.del) message.del = true; + + this.send(message); + } + + _sendOps(collection, id, ops) { + for (var i = 0; i < ops.length; i++) { + this._sendOp(collection, id, ops[i]); + } + } + + _sendOpsBulk(collection, opsMap) { + for (var id in opsMap) { + var ops = opsMap[id]; + this._sendOps(collection, id, ops); + } + } + + _reply(request, err, message) { + var agent = this; + var backend = agent.backend; + if (err) { + request.error = getReplyErrorObject(err); + agent.send(request); + return; + } + if (!message) message = {}; + + message.a = request.a; + if (request.id) { + message.id = request.id; + } else { + if (request.c) message.c = request.c; + if (request.d) message.d = request.d; + if (request.b && !message.data) message.b = request.b; + } + + var middlewareContext = {request: request, reply: message}; + backend.trigger(backend.MIDDLEWARE_ACTIONS.reply, agent, middlewareContext, function(err) { + if (err) { + request.error = getReplyErrorObject(err); + agent.send(request); + } else { + agent.send(middlewareContext.reply); + } + }); + } + + // Start processing events from the stream + _open() { + if (this.closed) return; + this.backend.agentsCount++; + if (!this.stream.isServer) this.backend.remoteAgentsCount++; + + var agent = this; + this.stream.on('data', function(chunk) { + if (agent.closed) return; + + if (typeof chunk !== 'object') { + var err = new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Received non-object message'); + return agent.close(err); + } + + var request = {data: chunk}; + agent.backend.trigger(agent.backend.MIDDLEWARE_ACTIONS.receive, agent, request, function(err) { + var callback = function(err, message) { + agent._reply(request.data, err, message); + }; + if (err) return callback(err); + agent._handleMessage(request.data, callback); + }); + }); + + var cleanup = agent._cleanup.bind(agent); + this.stream.on('end', cleanup); + this.stream.on('close', cleanup); + } + + // Check a request to see if its valid. Returns an error if there's a problem. + _checkRequest(request) { + if ( + request.a === ACTIONS.queryFetch || + request.a === ACTIONS.querySubscribe || + request.a === ACTIONS.queryUnsubscribe + ) { + // Query messages need an ID property. + if (typeof request.id !== 'number') return 'Missing query ID'; + } else if (request.a === ACTIONS.op || + request.a === ACTIONS.fetch || + request.a === ACTIONS.subscribe || + request.a === ACTIONS.unsubscribe || + request.a === ACTIONS.presence) { + // Doc-based request. + if (request.c != null) { + if (typeof request.c !== 'string' || util.isDangerousProperty(request.c)) { + return 'Invalid collection'; + } + } + if (request.d != null) { + if (typeof request.d !== 'string' || util.isDangerousProperty(request.d)) { + return 'Invalid id'; + } + } + + if (request.a === ACTIONS.op || request.a === ACTIONS.presence) { + if (request.v != null && (typeof request.v !== 'number' || request.v < 0)) return 'Invalid version'; + } + + if (request.a === ACTIONS.presence) { + if (typeof request.id !== 'string' || util.isDangerousProperty(request.id)) { + return 'Invalid presence ID'; + } + } + } else if ( + request.a === ACTIONS.bulkFetch || + request.a === ACTIONS.bulkSubscribe || + request.a === ACTIONS.bulkUnsubscribe + ) { + // Bulk request + if (request.c != null) { + if (typeof request.c !== 'string' || util.isDangerousProperty(request.c)) { + return 'Invalid collection'; + } + } + if (typeof request.b !== 'object') return 'Invalid bulk subscribe data'; + } + if (request.ch != null) { + if (typeof request.ch !== 'string' || util.isDangerousProperty(request.ch)) { + return 'Invalid presence channel'; + } + } + } + + // Handle an incoming message from the client + _handleMessage(request, callback) { + try { + var errMessage = this._checkRequest(request); + if (errMessage) return callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, errMessage)); + this._checkFirstMessage(request); + + switch (request.a) { + case ACTIONS.handshake: + if (request.id) this.src = request.id; + this._setProtocol(request); + return callback(null, this._initMessage(ACTIONS.handshake)); + case ACTIONS.queryFetch: + return this._queryFetch(request.id, request.c, request.q, getQueryOptions(request), callback); + case ACTIONS.querySubscribe: + return this._querySubscribe(request.id, request.c, request.q, getQueryOptions(request), callback); + case ACTIONS.queryUnsubscribe: + return this._queryUnsubscribe(request.id, callback); + case ACTIONS.bulkFetch: + return this._fetchBulk(request.c, request.b, callback); + case ACTIONS.bulkSubscribe: + return this._subscribeBulk(request.c, request.b, callback); + case ACTIONS.bulkUnsubscribe: + return this._unsubscribeBulk(request.c, request.b, callback); + case ACTIONS.fetch: + return this._fetch(request.c, request.d, request.v, callback); + case ACTIONS.subscribe: + return this._subscribe(request.c, request.d, request.v, callback); + case ACTIONS.unsubscribe: + return this._unsubscribe(request.c, request.d, callback); + case ACTIONS.op: + // Normalize the properties submitted + var op = createClientOp(request, this._src()); + if (op.seq >= util.MAX_SAFE_INTEGER) { + return callback(new ShareDBError( + ERROR_CODE.ERR_CONNECTION_SEQ_INTEGER_OVERFLOW, + 'Connection seq has exceeded the max safe integer, maybe from being open for too long' + )); + } + if (!op) return callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Invalid op message')); + return this._submit(request.c, request.d, op, callback); + case ACTIONS.snapshotFetch: + return this._fetchSnapshot(request.c, request.d, request.v, callback); + case ACTIONS.snapshotFetchByTimestamp: + return this._fetchSnapshotByTimestamp(request.c, request.d, request.ts, callback); + case ACTIONS.presence: + if (!this.backend.presenceEnabled) return; + var presence = this._createPresence(request); + if (presence.t && !util.supportsPresence(types.map[presence.t])) { + return callback({ + code: ERROR_CODE.ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE, + message: 'Type does not support presence: ' + presence.t + }); + } + return this._broadcastPresence(presence, callback); + case ACTIONS.presenceSubscribe: + if (!this.backend.presenceEnabled) return; + return this._subscribePresence(request.ch, request.seq, callback); + case ACTIONS.presenceUnsubscribe: + return this._unsubscribePresence(request.ch, request.seq, callback); + case ACTIONS.presenceRequest: + return this._requestPresence(request.ch, callback); + case ACTIONS.pingPong: + return this._pingPong(callback); + default: + callback(new ShareDBError(ERROR_CODE.ERR_MESSAGE_BADLY_FORMED, 'Invalid or unknown message')); + } + } catch (err) { + callback(err); + } + } + + _queryFetch(queryId, collection, query, options, callback) { + // Fetch the results of a query once + this.backend.queryFetch(this, collection, query, options, function(err, results, extra) { + if (err) return callback(err); + var message = { + data: getResultsData(results), + extra: extra + }; + callback(null, message); + }); + } + + _querySubscribe(queryId, collection, query, options, callback) { + // Subscribe to a query. The client is sent the query results and its + // notified whenever there's a change + var agent = this; + var wait = 1; + var message; + function finish(err) { + if (err) return callback(err); + if (--wait) return; + callback(null, message); + } + if (options.fetch) { + wait++; + this.backend.fetchBulk(this, collection, options.fetch, function(err, snapshotMap) { + if (err) return finish(err); + message = getMapResult(snapshotMap); + finish(); + }); + } + if (options.fetchOps) { + wait++; + this._fetchBulkOps(collection, options.fetchOps, finish); + } + this.backend.querySubscribe(this, collection, query, options, function(err, emitter, results, extra) { + if (err) return finish(err); + if (agent.closed) return emitter.destroy(); + + agent._subscribeToQuery(emitter, queryId, collection, query); + // No results are returned when ids are passed in as an option. Instead, + // want to re-poll the entire query once we've established listeners to + // emit any diff in results + if (!results) { + emitter.queryPoll(finish); + return; + } + message = { + data: getResultsData(results), + extra: extra + }; + finish(); + }); + } + + _pingPong(callback) { + var error = null; + var message = { + a: ACTIONS.pingPong + }; + callback(error, message); + } + + _queryUnsubscribe(queryId, callback) { + var emitter = this.subscribedQueries[queryId]; + if (emitter) { + emitter.destroy(); + delete this.subscribedQueries[queryId]; + } + util.nextTick(callback); + } + + _fetch(collection, id, version, callback) { + if (version == null) { + // Fetch a snapshot + this.backend.fetch(this, collection, id, function(err, snapshot) { + if (err) return callback(err); + callback(null, {data: getSnapshotData(snapshot)}); + }); + } else { + // It says fetch on the tin, but if a version is specified the client + // actually wants me to fetch some ops + this._fetchOps(collection, id, version, callback); + } + } + + _fetchOps(collection, id, version, callback) { + var agent = this; + this.backend.getOps(this, collection, id, version, null, function(err, ops) { + if (err) return callback(err); + agent._sendOps(collection, id, ops); + callback(); + }); + } + + _fetchBulk(collection, versions, callback) { + if (Array.isArray(versions)) { + this.backend.fetchBulk(this, collection, versions, function(err, snapshotMap) { + if (err) { + return callback(err); + } + if (snapshotMap) { + var result = getMapResult(snapshotMap); + callback(null, result); + } else { + callback(); + } + }); + } else { + this._fetchBulkOps(collection, versions, callback); + } + } + + _fetchBulkOps(collection, versions, callback) { + var agent = this; + this.backend.getOpsBulk(this, collection, versions, null, function(err, opsMap) { + if (err) return callback(err); + agent._sendOpsBulk(collection, opsMap); + callback(); + }); + } + + _subscribe(collection, id, version, callback) { + // If the version is specified, catch the client up by sending all ops + // since the specified version + var agent = this; + this.backend.subscribe(this, collection, id, version, function(err, stream, snapshot, ops) { + if (err) return callback(err); + // If we're subscribing from a known version, send any ops committed since + // the requested version to bring the client's doc up to date + if (ops) { + agent._sendOps(collection, id, ops); + } + // In addition, ops may already be queued on the stream by pubsub. + // Subscribe is called before the ops or snapshot are fetched, so it is + // possible that some ops may be duplicates. Clients should ignore any + // duplicate ops they may receive. This will flush ops already queued and + // subscribe to ongoing ops from the stream + agent._subscribeToStream(collection, id, stream); + // Snapshot is returned only when subscribing from a null version. + // Otherwise, ops will have been pushed into the stream + if (snapshot) { + callback(null, {data: getSnapshotData(snapshot)}); + } else { + callback(); + } + }); + } + + _subscribeBulk(collection, versions, callback) { + // See _subscribe() above. This function's logic should match but in bulk + var agent = this; + this.backend.subscribeBulk(this, collection, versions, function(err, streams, snapshotMap, opsMap) { + if (err) { + return callback(err); + } + if (opsMap) { + agent._sendOpsBulk(collection, opsMap); + } + for (var id in streams) { + agent._subscribeToStream(collection, id, streams[id]); + } + if (snapshotMap) { + var result = getMapResult(snapshotMap); + callback(null, result); + } else { + callback(); + } + }); + } + + _unsubscribe(collection, id, callback) { + // Unsubscribe from the specified document. This cancels the active + // stream or an inflight subscribing state + var docs = this.subscribedDocs[collection]; + var stream = docs && docs[id]; + if (stream) stream.destroy(); + util.nextTick(callback); + } + + _unsubscribeBulk(collection, ids, callback) { + var docs = this.subscribedDocs[collection]; + if (!docs) return util.nextTick(callback); + for (var i = 0; i < ids.length; i++) { + var id = ids[i]; + var stream = docs[id]; + if (stream) stream.destroy(); + } + util.nextTick(callback); + } + + _submit(collection, id, op, callback) { + var agent = this; + this.backend.submit(this, collection, id, op, null, function(err, ops, request) { + // Message to acknowledge the op was successfully submitted + var ack = {src: op.src, seq: op.seq, v: op.v}; + if (request._fixupOps.length) ack[ACTIONS.fixup] = request._fixupOps; + if (err) { + // Occasional 'Op already submitted' errors are expected to happen as + // part of normal operation, since inflight ops need to be resent after + // disconnect. In this case, ack the op so the client can proceed + if (err.code === ERROR_CODE.ERR_OP_ALREADY_SUBMITTED) return callback(null, ack); + return callback(err); + } + + // Reply with any operations that the client is missing. + agent._sendOps(collection, id, ops); + callback(null, ack); + }); + } + + _fetchSnapshot(collection, id, version, callback) { + this.backend.fetchSnapshot(this, collection, id, version, callback); + } + + _fetchSnapshotByTimestamp(collection, id, timestamp, callback) { + this.backend.fetchSnapshotByTimestamp(this, collection, id, timestamp, callback); + } + + _initMessage(action) { + return { + a: action, + protocol: protocol.major, + protocolMinor: protocol.minor, + id: this._src(), + type: types.defaultType.uri + }; + } + + _src() { + return this.src || this.clientId; + } + + _broadcastPresence(presence, callback) { + var agent = this; + var backend = this.backend; + var presenceRequests = this.presenceRequests; + var context = { + presence: presence, + collection: presence.c + }; + var start = Date.now(); + + var subscriptionUpdater = presence.p === null ? + this._unsubscribeDocVersion.bind(this) : + this._subscribeDocVersion.bind(this); + + subscriptionUpdater(presence.c, presence.d, function(error) { + if (error) return callback(error); + backend.trigger(backend.MIDDLEWARE_ACTIONS.receivePresence, agent, context, function(error) { + if (error) return callback(error); + var requests = presenceRequests[presence.ch] || (presenceRequests[presence.ch] = Object.create(null)); + var previousRequest = requests[presence.id]; + if (!previousRequest || previousRequest.pv < presence.pv) { + presenceRequests[presence.ch][presence.id] = presence; + } + + var transformer = function(agent, presence, callback) { + callback(null, presence); + }; + + var latestDocVersion = util.dig(agent.latestDocVersions, presence.c, presence.d); + var presenceIsUpToDate = presence.v === latestDocVersion; + if (!presenceIsUpToDate) { + transformer = backend.transformPresenceToLatestVersion.bind(backend); + } + + transformer(agent, presence, function(error, presence) { + if (error) return callback(error); + var channel = agent._getPresenceChannel(presence.ch); + agent.backend.pubsub.publish([channel], presence, function(error) { + if (error) return callback(error); + backend.emit('timing', 'presence.broadcast', Date.now() - start, context); + callback(null, presence); + }); + }); + }); + }); + } + + _subscribeDocVersion(collection, id, callback) { + if (!collection || !id) return callback(); + + var latestDocVersions = this.latestDocVersions; + var isSubscribed = util.dig(latestDocVersions, collection, id) !== undefined; + if (isSubscribed) return callback(); + + var agent = this; + this.backend.subscribe(this, collection, id, null, function(error, stream, snapshot) { + if (error) return callback(error); + + var versions = latestDocVersions[collection] || (latestDocVersions[collection] = Object.create(null)); + versions[id] = snapshot.v; + + agent._subscribeMapToStream(agent.latestDocVersionStreams, collection, id, stream, function(op) { + // op.v behind snapshot.v by 1 + latestDocVersions[collection][id] = op.v + 1; + }); + + callback(); + }); + } + + _unsubscribeDocVersion(collection, id, callback) { + var stream = util.dig(this.latestDocVersionStreams, collection, id); + if (stream) stream.destroy(); + util.digAndRemove(this.latestDocVersions, collection, id); + util.nextTick(callback); + } + + _createPresence(request) { + return { + a: ACTIONS.presence, + ch: request.ch, + src: this._src(), + id: request.id, // Presence ID, not Doc ID (which is 'd') + p: request.p, + pv: request.pv, + // The c,d,v,t fields are only set for DocPresence + c: request.c, + d: request.d, + v: request.v, + t: request.t + }; + } + + _subscribePresence(channel, seq, cb) { + var agent = this; + + function callback(error) { + cb(error, {ch: channel, seq: seq}); + } + + var existingStream = agent.subscribedPresences[channel]; + if (existingStream) { + agent.presenceSubscriptionSeq[channel] = seq; + return callback(); + } + + var presenceChannel = this._getPresenceChannel(channel); + this.backend.pubsub.subscribe(presenceChannel, function(error, stream) { + if (error) return callback(error); + if (seq < agent.presenceSubscriptionSeq[channel]) { + stream.destroy(); + return callback(); + } + agent.presenceSubscriptionSeq[channel] = seq; + agent.subscribedPresences[channel] = stream; + agent._subscribeToPresenceStream(channel, stream); + agent._requestPresence(channel, function(error) { + callback(error); + }); + }); + } + + _unsubscribePresence(channel, seq, callback) { + this.presenceSubscriptionSeq[channel] = seq; + var stream = this.subscribedPresences[channel]; + if (stream) stream.destroy(); + delete this.subscribedPresences[channel]; + callback(null, {ch: channel, seq: seq}); + } + + _getPresenceChannel(channel) { + return '$presence.' + channel; + } + + _requestPresence(channel, callback) { + var presenceChannel = this._getPresenceChannel(channel); + this.backend.pubsub.publish([presenceChannel], {ch: channel, r: true, src: this.clientId}, callback); + } + + _handlePresenceData(presence) { + if (presence.src === this._src()) return; + + if (presence.r) return this.send({a: ACTIONS.presenceRequest, ch: presence.ch}); + + var backend = this.backend; + var context = { + collection: presence.c, + presence: presence + }; + var agent = this; + backend.trigger(backend.MIDDLEWARE_ACTIONS.sendPresence, this, context, function(error) { + if (error) { + if (backend.doNotForwardSendPresenceErrorsToClient) backend.errorHandler(error, {agent: agent}); + else agent.send({a: ACTIONS.presence, ch: presence.ch, id: presence.id, error: getReplyErrorObject(error)}); + return; + } + agent.send(presence); + }); + } + + _checkFirstMessage(request) { + if (this._handshakeReceived) return; + if (!this._firstReceivedMessage) this._firstReceivedMessage = request; + + if (request.a === ACTIONS.handshake) { + this._handshakeReceived = true; + if (this._firstReceivedMessage.a !== ACTIONS.handshake) { + logger.warn('Unexpected message received before handshake', this._firstReceivedMessage); + } + // Release memory + this._firstReceivedMessage = null; + } + } + + _setProtocol(request) { + this.protocol.major = request.protocol; + this.protocol.minor = request.protocolMinor; + } +} + +export = Agent; + +function getReplyErrorObject(err) { + if (typeof err === 'string') { + return { + code: ERROR_CODE.ERR_UNKNOWN_ERROR, + message: err + }; + } else { + if (err.stack) { + logger.info(err.stack); + } + return { + code: err.code, + message: err.message + }; + } +} + +function getQueryOptions(request) { + var results = request.r; + var ids; + var fetch; + var fetchOps; + if (results) { + ids = []; + for (var i = 0; i < results.length; i++) { + var result = results[i]; + var id = result[0]; + var version = result[1]; + ids.push(id); + if (version == null) { + if (fetch) { + fetch.push(id); + } else { + fetch = [id]; + } + } else { + if (!fetchOps) fetchOps = Object.create(null); + fetchOps[id] = version; + } + } + } + var options = request.o || {}; + options.ids = ids; + options.fetch = fetch; + options.fetchOps = fetchOps; + return options; +} + +function getResultsData(results) { + var items = []; + for (var i = 0; i < results.length; i++) { + var result = results[i]; + var item = getSnapshotData(result); + item.d = result.id; + items.push(item); + } + return items; +} + +function getMapResult(snapshotMap) { + var data = Object.create(null); + for (var id in snapshotMap) { + var mapValue = snapshotMap[id]; + // fetchBulk / subscribeBulk map data can have either a Snapshot or an object + // `{error: Error | string}` as a value. + if (mapValue.error) { + // Transform errors to serialization-friendly objects. + data[id] = {error: getReplyErrorObject(mapValue.error)}; + } else { + data[id] = getSnapshotData(mapValue); + } + } + return {data: data}; +} + +function getSnapshotData(snapshot) { + var data = { + v: snapshot.v, + data: snapshot.data + }; + if (types.defaultType !== types.map[snapshot.type]) { + data.type = snapshot.type; + } + return data; +} + +function createClientOp(request, clientId) { + // src can be provided if it is not the same as the current agent, + // such as a resubmission after a reconnect, but it usually isn't needed + var src = request.src || clientId; + // c, d, and m arguments are intentionally undefined. These are set later + return ('op' in request) ? new EditOp(src, request.seq, request.v, request.op, request.x) : + (request.create) ? new CreateOp(src, request.seq, request.v, request.create, request.x) : + (request.del) ? new DeleteOp(src, request.seq, request.v, request.del, request.x) : + undefined; +} + +function shallowCopy(object) { + var out = {}; + for (var key in object) { + if (util.hasOwn(object, key)) { + out[key] = object[key]; + } + } + return out; +} + +class CreateOp { + src; + seq; + v; + create; + c; + d; + m; + x; + + constructor(src, seq, v, create, x, c, d, m) { + this.src = src; + this.seq = seq; + this.v = v; + this.create = create; + this.c = c; + this.d = d; + this.m = m; + this.x = x; + } +} + +class EditOp { + src; + seq; + v; + op; + c; + d; + m; + x; + + constructor(src, seq, v, op, x, c, d, m) { + this.src = src; + this.seq = seq; + this.v = v; + this.op = op; + this.c = c; + this.d = d; + this.m = m; + this.x = x; + } +} + +class DeleteOp { + src; + seq; + v; + del; + c; + d; + m; + x; + + constructor(src, seq, v, del, x, c, d, m) { + this.src = src; + this.seq = seq; + this.v = v; + this.del = del; + this.c = c; + this.d = d; + this.m = m; + this.x = x; + } +} diff --git a/src/backend.ts b/src/backend.ts new file mode 100644 index 000000000..87a6a9b97 --- /dev/null +++ b/src/backend.ts @@ -0,0 +1,972 @@ +import async = require('async'); +import Agent = require('./agent'); +import Connection = require('./client/connection'); +import emitter = require('./emitter'); +import MemoryDB = require('./db/memory'); +import NoOpMilestoneDB = require('./milestone-db/no-op'); +import MemoryPubSub = require('./pubsub/memory'); +import ot = require('./ot'); +import projections = require('./projections'); +import QueryEmitter = require('./query-emitter'); +import ShareDBError = require('./error'); +import Snapshot = require('./snapshot'); +import StreamSocket = require('./stream-socket'); +import SubmitRequest = require('./submit-request'); +import ReadSnapshotsRequest = require('./read-snapshots-request'); +import util = require('./util'); +import logger = require('./logger'); + +var ERROR_CODE = ShareDBError.CODES; + +class Backend { + db; + pubsub; + extraDbs; + milestoneDb; + projections; + suppressPublish; + maxSubmitRetries; + presenceEnabled; + doNotForwardSendPresenceErrorsToClient; + doNotCommitNoOps; + middleware; + agentsCount; + remoteAgentsCount; + errorHandler; + + constructor(options) { + if (!(this instanceof Backend)) return new Backend(options); + emitter.EventEmitter.call(this); + + if (!options) options = {}; + this.db = options.db || new MemoryDB(); + this.pubsub = options.pubsub || new MemoryPubSub(); + // This contains any extra databases that can be queried + this.extraDbs = options.extraDbs || {}; + this.milestoneDb = options.milestoneDb || new NoOpMilestoneDB(); + + // Map from projected collection -> {type, fields} + this.projections = Object.create(null); + + this.suppressPublish = !!options.suppressPublish; + this.maxSubmitRetries = options.maxSubmitRetries || null; + this.presenceEnabled = !!options.presence; + this.doNotForwardSendPresenceErrorsToClient = !!options.doNotForwardSendPresenceErrorsToClient; + if (this.presenceEnabled && !this.doNotForwardSendPresenceErrorsToClient) { + logger.warn( + 'Broadcasting "sendPresence" middleware errors to clients is deprecated ' + + 'and will be removed in a future release. Disable this behaviour with:\n\n' + + 'new Backend({doNotForwardSendPresenceErrorsToClient: true})\n\n' + ); + } + this.doNotCommitNoOps = !!options.doNotCommitNoOps; + + // Map from event name to a list of middleware + this.middleware = Object.create(null); + + // The number of open agents for monitoring and testing memory leaks + this.agentsCount = 0; + this.remoteAgentsCount = 0; + + this.errorHandler = typeof options.errorHandler === 'function' ? + options.errorHandler : + // eslint-disable-next-line no-unused-vars + function(error, context) { + logger.error(error); + }; + } + + close(callback) { + var wait = 4; + var backend = this; + function finish(err) { + if (err) { + if (callback) return callback(err); + return backend.emit('error', err); + } + if (--wait) return; + if (callback) callback(); + } + this.pubsub.close(finish); + this.db.close(finish); + this.milestoneDb.close(finish); + for (var name in this.extraDbs) { + wait++; + this.extraDbs[name].close(finish); + } + finish(); + } + + connect(connection, req, callback) { + var socket = new StreamSocket(); + if (connection) { + connection.bindToSocket(socket); + } else { + connection = new Connection(socket); + } + socket._open(); + var agent = this.listen(socket.stream, req); + // Store a reference to the agent on the connection for convenience. This is + // not used internal to ShareDB, but it is handy for server-side only user + // code that may cache state on the agent and read it in middleware + connection.agent = agent; + + if (typeof callback === 'function') { + connection.once('connected', function() { + callback(connection); + }); + } + + return connection; + } + + /** A client has connected through the specified stream. Listen for messages. + * + * The optional second argument (req) is an initial request which is passed + * through to any connect() middleware. This is useful for inspecting cookies + * or an express session or whatever on the request object in your middleware. + * + * (The agent is available through all middleware) + */ + listen(stream, req) { + var agent = new Agent(this, stream); + this.trigger(this.MIDDLEWARE_ACTIONS.connect, agent, {stream: stream, req: req}, function(err) { + if (err) return agent.close(err); + agent._open(); + }); + return agent; + } + + addProjection(name, collection, fields) { + if (this.projections[name]) { + throw new Error('Projection ' + name + ' already exists'); + } + + for (var key in fields) { + if (fields[key] !== true) { + throw new Error('Invalid field ' + key + ' - fields must be {somekey: true}. Subfields not currently supported.'); + } + } + + this.projections[name] = { + target: collection, + fields: fields + }; + } + + /** + * Add middleware to an action or array of actions + */ + use(action, fn) { + if (Array.isArray(action)) { + for (var i = 0; i < action.length; i++) { + this.use(action[i], fn); + } + return this; + } + var fns = this.middleware[action] || (this.middleware[action] = []); + fns.push(fn); + return this; + } + + /** + * Passes request through the middleware stack + * + * Middleware may modify the request object. After all middleware have been + * invoked we call `callback` with `null` and the modified request. If one of + * the middleware resturns an error the callback is called with that error. + */ + trigger(action, agent, request, callback) { + request.action = action; + request.agent = agent; + request.backend = this; + + var fns = this.middleware[action]; + if (!fns) return callback(); + + // Copying the triggers we'll fire so they don't get edited while we iterate. + fns = fns.slice(); + var next = function(err) { + if (err) return callback(err); + var fn = fns.shift(); + if (!fn) return callback(); + fn(request, next); + }; + next(); + } + + // Submit an operation on the named collection/docname. op should contain a + // {op:}, {create:} or {del:} field. It should probably contain a v: field (if + // it doesn't, it defaults to the current version). + submit(agent, index, id, op, options, originalCallback) { + var backend = this; + var request = new SubmitRequest(this, agent, index, id, op, options); + + var callback = function(error, ops) { + backend.emit('submitRequestEnd', error, request); + originalCallback(error, ops, request); + }; + + var err = ot.checkOp(op); + if (err) return callback(err); + backend.trigger(backend.MIDDLEWARE_ACTIONS.submit, agent, request, function(err) { + if (err) return callback(err); + request.submit(function(err) { + if (err) return callback(err); + backend.trigger(backend.MIDDLEWARE_ACTIONS.afterWrite, agent, request, function(err) { + if (err) return callback(err); + backend._sanitizeOps(agent, request.projection, request.collection, id, request.ops, function(err) { + if (err) return callback(err); + backend.emit('timing', 'submit.total', Date.now() - request.start, request); + callback(err, request.ops); + }); + }); + }); + }); + } + + sanitizeOp(agent, index, id, op, callback) { + var projection = this.projections[index]; + var collection = (projection) ? projection.target : index; + this._sanitizeOp(agent, projection, collection, id, op, callback); + } + + _sanitizeOp(agent, projection, collection, id, op, callback) { + if (projection) { + try { + projections.projectOp(projection.fields, op); + } catch (err) { + return callback(err); + } + } + this.trigger(this.MIDDLEWARE_ACTIONS.op, agent, {collection: collection, id: id, op: op}, callback); + } + + _sanitizeOps(agent, projection, collection, id, ops, callback) { + var backend = this; + async.each(ops, function(op, eachCb) { + backend._sanitizeOp(agent, projection, collection, id, op, function(err) { + util.nextTick(eachCb, err); + }); + }, callback); + } + + _sanitizeOpsBulk(agent, projection, collection, opsMap, callback) { + var backend = this; + async.forEachOf(opsMap, function(ops, id, eachCb) { + backend._sanitizeOps(agent, projection, collection, id, ops, eachCb); + }, callback); + } + + _sanitizeSnapshots(agent, projection, collection, snapshots, snapshotType, callback) { + if (projection) { + try { + projections.projectSnapshots(projection.fields, snapshots); + } catch (err) { + return callback(err); + } + } + + var request = new ReadSnapshotsRequest(collection, snapshots, snapshotType); + + this.trigger(this.MIDDLEWARE_ACTIONS.readSnapshots, agent, request, function(err) { + if (err) return callback(err); + // Handle "partial rejection" - "readSnapshots" middleware functions can use + // `request.rejectSnapshotRead(snapshot, error)` to reject the read of a specific snapshot. + if (request.hasSnapshotRejection()) { + err = request.getReadSnapshotsError(); + } + if (err) { + callback(err); + } else { + callback(); + } + }); + } + + _getSnapshotProjection(db, projection) { + return (db.projectsSnapshots) ? null : projection; + } + + _getSnapshotsFromMap(ids, snapshotMap) { + var snapshots = new Array(ids.length); + for (var i = 0; i < ids.length; i++) { + var id = ids[i]; + snapshots[i] = snapshotMap[id]; + } + return snapshots; + } + + _getSanitizedOps(agent, projection, collection, id, from, to, opsOptions, callback) { + var backend = this; + if (!opsOptions) opsOptions = {}; + if (agent) opsOptions.agentCustom = agent.custom; + backend.db.getOps(collection, id, from, to, opsOptions, function(err, ops) { + if (err) return callback(err); + backend._sanitizeOps(agent, projection, collection, id, ops, function(err) { + if (err) return callback(err); + callback(null, ops); + }); + }); + } + + _getSanitizedOpsBulk(agent, projection, collection, fromMap, toMap, opsOptions, callback) { + var backend = this; + if (!opsOptions) opsOptions = {}; + if (agent) opsOptions.agentCustom = agent.custom; + backend.db.getOpsBulk(collection, fromMap, toMap, opsOptions, function(err, opsMap) { + if (err) return callback(err); + backend._sanitizeOpsBulk(agent, projection, collection, opsMap, function(err) { + if (err) return callback(err); + callback(null, opsMap); + }); + }); + } + + // Non inclusive - gets ops from [from, to). Ie, all relevant ops. If to is + // not defined (null or undefined) then it returns all ops. + getOps(agent, index, id, from, to, options, callback) { + if (typeof options === 'function') { + callback = options; + options = null; + } + var start = Date.now(); + var projection = this.projections[index]; + var collection = (projection) ? projection.target : index; + var backend = this; + var request = { + agent: agent, + index: index, + collection: collection, + id: id, + from: from, + to: to + }; + var opsOptions = options && options.opsOptions; + backend._getSanitizedOps(agent, projection, collection, id, from, to, opsOptions, function(err, ops) { + if (err) return callback(err); + backend.emit('timing', 'getOps', Date.now() - start, request); + callback(null, ops); + }); + } + + getOpsBulk(agent, index, fromMap, toMap, options, callback) { + if (typeof options === 'function') { + callback = options; + options = null; + } + var start = Date.now(); + var projection = this.projections[index]; + var collection = (projection) ? projection.target : index; + var backend = this; + var request = { + agent: agent, + index: index, + collection: collection, + fromMap: fromMap, + toMap: toMap + }; + var opsOptions = options && options.opsOptions; + backend._getSanitizedOpsBulk(agent, projection, collection, fromMap, toMap, opsOptions, function(err, opsMap) { + if (err) return callback(err); + backend.emit('timing', 'getOpsBulk', Date.now() - start, request); + callback(null, opsMap); + }); + } + + fetch(agent, index, id, options, callback) { + if (typeof options === 'function') { + callback = options; + options = null; + } + var start = Date.now(); + var projection = this.projections[index]; + var collection = (projection) ? projection.target : index; + var fields = projection && projection.fields; + var backend = this; + var request = { + agent: agent, + index: index, + collection: collection, + id: id + }; + var snapshotOptions = (options && options.snapshotOptions) || {}; + snapshotOptions.agentCustom = agent.custom; + backend.db.getSnapshot(collection, id, fields, snapshotOptions, function(err, snapshot) { + if (err) return callback(err); + var snapshotProjection = backend._getSnapshotProjection(backend.db, projection); + var snapshots = [snapshot]; + backend._sanitizeSnapshots( + agent, + snapshotProjection, + collection, + snapshots, + backend.SNAPSHOT_TYPES.current, + function(err) { + if (err) return callback(err); + backend.emit('timing', 'fetch', Date.now() - start, request); + callback(null, snapshot); + }); + }); + } + + /** + * Map of document id to Snapshot or error object. + * @typedef {{ [id: string]: Snapshot | { error: Error | string } }} SnapshotMap + */ + + /** + * @param {Agent} agent + * @param {string} index + * @param {string[]} ids + * @param {*} options + * @param {(err?: Error | string, snapshotMap?: SnapshotMap) => void} callback + */ + fetchBulk(agent, index, ids, options, callback) { + if (typeof options === 'function') { + callback = options; + options = null; + } + var start = Date.now(); + var projection = this.projections[index]; + var collection = (projection) ? projection.target : index; + var fields = projection && projection.fields; + var backend = this; + var request = { + agent: agent, + index: index, + collection: collection, + ids: ids + }; + var snapshotOptions = (options && options.snapshotOptions) || {}; + snapshotOptions.agentCustom = agent.custom; + backend.db.getSnapshotBulk(collection, ids, fields, snapshotOptions, function(err, snapshotMap) { + if (err) return callback(err); + var snapshotProjection = backend._getSnapshotProjection(backend.db, projection); + var snapshots = backend._getSnapshotsFromMap(ids, snapshotMap); + backend._sanitizeSnapshots( + agent, + snapshotProjection, + collection, + snapshots, + backend.SNAPSHOT_TYPES.current, + function(err) { + if (err) { + if (err.code === ERROR_CODE.ERR_SNAPSHOT_READS_REJECTED) { + for (var docId in err.idToError) { + snapshotMap[docId] = {error: err.idToError[docId]}; + } + err = undefined; + } else { + snapshotMap = undefined; + } + } + backend.emit('timing', 'fetchBulk', Date.now() - start, request); + callback(err, snapshotMap); + }); + }); + } + + // Subscribe to the document from the specified version or null version + subscribe(agent, index, id, version, options, callback) { + if (typeof options === 'function') { + callback = options; + options = null; + } + if (options) { + // We haven't yet implemented the ability to pass options to subscribe. This is because we need to + // add the ability to SubmitRequest.commit to optionally pass the metadata to other clients on + // PubSub. This behaviour is not needed right now, but we have added an options object to the + // subscribe() signature so that it remains consistent with getOps() and fetch(). + return callback(new ShareDBError( + ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, + 'Passing options to subscribe has not been implemented' + )); + } + var start = Date.now(); + var projection = this.projections[index]; + var collection = (projection) ? projection.target : index; + var channel = this.getDocChannel(collection, id); + var backend = this; + var request = { + agent: agent, + index: index, + collection: collection, + id: id, + version: version + }; + backend.pubsub.subscribe(channel, function(err, stream) { + if (err) return callback(err); + if (version == null) { + // Subscribing from null means that the agent doesn't have a document + // and needs to fetch it as well as subscribing + backend.fetch(agent, index, id, function(err, snapshot) { + if (err) { + stream.destroy(); + return callback(err); + } + backend.emit('timing', 'subscribe.snapshot', Date.now() - start, request); + callback(null, stream, snapshot); + }); + } else { + backend._getSanitizedOps(agent, projection, collection, id, version, null, null, function(err, ops) { + if (err) { + stream.destroy(); + return callback(err); + } + backend.emit('timing', 'subscribe.ops', Date.now() - start, request); + callback(null, stream, null, ops); + }); + } + }); + } + + /** + * Map of document id to pubsub stream. + * @typedef {{ [id: string]: Stream }} StreamMap + */ + /** + * Map of document id to array of ops for the doc. + * @typedef {{ [id: string]: Op[] }} OpsMap + */ + + /** + * @param {Agent} agent + * @param {string} index + * @param {string[]} versions + * @param {( + * err?: Error | string | null, + * streams?: StreamMap, + * snapshotMap?: SnapshotMap | null + * opsMap?: OpsMap + * ) => void} callback + */ + subscribeBulk(agent, index, versions, callback) { + var start = Date.now(); + var projection = this.projections[index]; + var collection = (projection) ? projection.target : index; + var backend = this; + var streams = Object.create(null); + var doFetch = Array.isArray(versions); + var ids = (doFetch) ? versions : Object.keys(versions); + var request = { + agent: agent, + index: index, + collection: collection, + versions: versions + }; + async.each(ids, function(id, eachCb) { + var channel = backend.getDocChannel(collection, id); + backend.pubsub.subscribe(channel, function(err, stream) { + if (err) return eachCb(err); + streams[id] = stream; + eachCb(); + }); + }, function(err) { + if (err) { + destroyStreams(streams); + return callback(err); + } + if (doFetch) { + // If an array of ids, get current snapshots + backend.fetchBulk(agent, index, ids, function(err, snapshotMap) { + if (err) { + // Full error, destroy all streams. + destroyStreams(streams); + streams = undefined; + snapshotMap = undefined; + } + for (var docId in snapshotMap) { + // The doc id could map to an object `{error: Error | string}`, which indicates that + // particular snapshot's read was rejected. Destroy the streams fur such docs. + if (snapshotMap[docId].error) { + streams[docId].destroy(); + delete streams[docId]; + } + } + backend.emit('timing', 'subscribeBulk.snapshot', Date.now() - start, request); + callback(err, streams, snapshotMap); + }); + } else { + // If a versions map, get ops since requested versions + backend._getSanitizedOpsBulk(agent, projection, collection, versions, null, null, function(err, opsMap) { + if (err) { + destroyStreams(streams); + return callback(err); + } + backend.emit('timing', 'subscribeBulk.ops', Date.now() - start, request); + callback(null, streams, null, opsMap); + }); + } + }); + } + + queryFetch(agent, index, query, options, callback) { + var start = Date.now(); + var backend = this; + backend._triggerQuery(agent, index, query, options, function(err, request) { + if (err) return callback(err); + backend._query(agent, request, function(err, snapshots, extra) { + if (err) return callback(err); + backend.emit('timing', 'queryFetch', Date.now() - start, request); + callback(null, snapshots, extra); + }); + }); + } + + // Options can contain: + // db: The name of the DB (if the DB is specified in the otherDbs when the backend instance is created) + // skipPoll: function(collection, id, op, query) {return true or false; } + // this is a synchronous function which can be used as an early filter for + // operations going through the system to reduce the load on the DB. + // pollDebounce: Minimum delay between subsequent database polls. This is + // used to batch updates to reduce load on the database at the expense of + // liveness + querySubscribe(agent, index, query, options, callback) { + var start = Date.now(); + var backend = this; + backend._triggerQuery(agent, index, query, options, function(err, request) { + if (err) return callback(err); + if (request.db.disableSubscribe) { + return callback(new ShareDBError( + ERROR_CODE.ERR_DATABASE_DOES_NOT_SUPPORT_SUBSCRIBE, + 'DB does not support subscribe' + )); + } + + var channels = request.channels; + + if (request.channel) { + logger.warn( + '[DEPRECATED] "query" middleware\'s context.channel is deprecated, use context.channels instead. ' + + 'Read more: https://share.github.io/sharedb/middleware/actions#query' + ); + channels = [request.channel]; + } + + if (!channels || !channels.length) { + return callback(new ShareDBError(ERROR_CODE.ERR_QUERY_CHANNEL_MISSING, 'Required minimum one query channel.')); + } + + var streams = []; + + function destroyStreams() { + streams.forEach(function(stream) { + stream.destroy(); + }); + } + + function createQueryEmitter() { + if (options.ids) { + var queryEmitter = new QueryEmitter(request, streams, options.ids); + backend.emit('timing', 'querySubscribe.reconnect', Date.now() - start, request); + callback(null, queryEmitter); + return; + } + // Issue query on db to get our initial results + backend._query(agent, request, function(err, snapshots, extra) { + if (err) { + destroyStreams(); + return callback(err); + } + var ids = pluckIds(snapshots); + var queryEmitter = new QueryEmitter(request, streams, ids, extra); + backend.emit('timing', 'querySubscribe.initial', Date.now() - start, request); + callback(null, queryEmitter, snapshots, extra); + }); + } + + channels.forEach(function(channel) { + backend.pubsub.subscribe(channel, function(err, stream) { + if (err) { + destroyStreams(); + return callback(err); + } + streams.push(stream); + + var subscribedToAllChannels = streams.length === channels.length; + if (subscribedToAllChannels) { + createQueryEmitter(); + } + }); + }); + }); + } + + _triggerQuery(agent, index, query, options, callback) { + var projection = this.projections[index]; + var collection = (projection) ? projection.target : index; + var fields = projection && projection.fields; + var request = { + index: index, + collection: collection, + projection: projection, + fields: fields, + channels: [this.getCollectionChannel(collection)], + query: query, + options: options, + db: null, + snapshotProjection: null + }; + var backend = this; + backend.trigger(backend.MIDDLEWARE_ACTIONS.query, agent, request, function(err) { + if (err) return callback(err); + // Set the DB reference for the request after the middleware trigger so + // that the db option can be changed in middleware + request.db = (options.db) ? backend.extraDbs[options.db] : backend.db; + if (!request.db) return callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_ADAPTER_NOT_FOUND, 'DB not found')); + request.snapshotProjection = backend._getSnapshotProjection(request.db, projection); + callback(null, request); + }); + } + + _query(agent, request, callback) { + var backend = this; + request.db.query(request.collection, request.query, request.fields, request.options, function(err, snapshots, extra) { + if (err) return callback(err); + backend._sanitizeSnapshots( + agent, + request.snapshotProjection, + request.collection, + snapshots, + backend.SNAPSHOT_TYPES.current, + function(err) { + callback(err, snapshots, extra); + }); + }); + } + + getCollectionChannel(collection) { + return collection; + } + + getDocChannel(collection, id) { + return collection + '.' + id; + } + + getChannels(collection, id) { + return [ + this.getCollectionChannel(collection), + this.getDocChannel(collection, id) + ]; + } + + fetchSnapshot(agent, index, id, version, callback) { + var start = Date.now(); + var backend = this; + var projection = this.projections[index]; + var collection = projection ? projection.target : index; + var request = { + agent: agent, + index: index, + collection: collection, + id: id, + version: version + }; + + this._fetchSnapshot(collection, id, version, function(error, snapshot) { + if (error) return callback(error); + var snapshotProjection = backend._getSnapshotProjection(backend.db, projection); + var snapshots = [snapshot]; + var snapshotType = backend.SNAPSHOT_TYPES.byVersion; + backend._sanitizeSnapshots(agent, snapshotProjection, collection, snapshots, snapshotType, function(error) { + if (error) return callback(error); + backend.emit('timing', 'fetchSnapshot', Date.now() - start, request); + callback(null, snapshot); + }); + }); + } + + _fetchSnapshot(collection, id, version, callback) { + var db = this.db; + var backend = this; + + var shouldGetLatestSnapshot = version === null; + if (shouldGetLatestSnapshot) { + return backend.db.getSnapshot(collection, id, null, null, function(error, snapshot) { + if (error) return callback(error); + + callback(null, snapshot); + }); + } + + + this.milestoneDb.getMilestoneSnapshot(collection, id, version, function(error, milestoneSnapshot) { + if (error) return callback(error); + + // Bypass backend.getOps so that we don't call _sanitizeOps. We want to avoid this, because: + // - we want to avoid the 'op' middleware, because we later use the 'readSnapshots' middleware in _sanitizeSnapshots + // - we handle the projection in _sanitizeSnapshots + var from = milestoneSnapshot ? milestoneSnapshot.v : 0; + db.getOps(collection, id, from, version, null, function(error, ops) { + if (error) return callback(error); + + backend._buildSnapshotFromOps(id, milestoneSnapshot, ops, function(error, snapshot) { + if (error) return callback(error); + + if (version > snapshot.v) { + return callback(new ShareDBError( + ERROR_CODE.ERR_OP_VERSION_NEWER_THAN_CURRENT_SNAPSHOT, + 'Requested version exceeds latest snapshot version' + )); + } + + callback(null, snapshot); + }); + }); + }); + } + + fetchSnapshotByTimestamp(agent, index, id, timestamp, callback) { + var start = Date.now(); + var backend = this; + var projection = this.projections[index]; + var collection = projection ? projection.target : index; + var request = { + agent: agent, + index: index, + collection: collection, + id: id, + timestamp: timestamp + }; + + this._fetchSnapshotByTimestamp(collection, id, timestamp, function(error, snapshot) { + if (error) return callback(error); + var snapshotProjection = backend._getSnapshotProjection(backend.db, projection); + var snapshots = [snapshot]; + var snapshotType = backend.SNAPSHOT_TYPES.byTimestamp; + backend._sanitizeSnapshots(agent, snapshotProjection, collection, snapshots, snapshotType, function(error) { + if (error) return callback(error); + backend.emit('timing', 'fetchSnapshot', Date.now() - start, request); + callback(null, snapshot); + }); + }); + } + + _fetchSnapshotByTimestamp(collection, id, timestamp, callback) { + var db = this.db; + var milestoneDb = this.milestoneDb; + var backend = this; + + var milestoneSnapshot; + var from = 0; + var to = null; + + var shouldGetLatestSnapshot = timestamp === null; + if (shouldGetLatestSnapshot) { + return backend.db.getSnapshot(collection, id, null, null, function(error, snapshot) { + if (error) return callback(error); + + callback(null, snapshot); + }); + } + + milestoneDb.getMilestoneSnapshotAtOrBeforeTime(collection, id, timestamp, function(error, snapshot) { + if (error) return callback(error); + milestoneSnapshot = snapshot; + if (snapshot) from = snapshot.v; + + milestoneDb.getMilestoneSnapshotAtOrAfterTime(collection, id, timestamp, function(error, snapshot) { + if (error) return callback(error); + if (snapshot) to = snapshot.v; + + var options = {metadata: true}; + db.getOps(collection, id, from, to, options, function(error, ops) { + if (error) return callback(error); + filterOpsInPlaceBeforeTimestamp(ops, timestamp); + backend._buildSnapshotFromOps(id, milestoneSnapshot, ops, callback); + }); + }); + }); + } + + _buildSnapshotFromOps(id, startingSnapshot, ops, callback) { + var snapshot = util.clone(startingSnapshot) || new Snapshot(id, 0, null, undefined, null); + var error = ot.applyOps(snapshot, ops, {_normalizeLegacyJson0Ops: true}); + callback(error, snapshot); + } + + transformPresenceToLatestVersion(agent, presence, callback) { + if (!presence.c || !presence.d) return callback(null, presence); + this.getOps(agent, presence.c, presence.d, presence.v, null, function(error, ops) { + if (error) return callback(error); + for (var i = 0; i < ops.length; i++) { + var op = ops[i]; + var isOwnOp = op.src === presence.src; + var transformError = ot.transformPresence(presence, op, isOwnOp); + if (transformError) { + return callback(transformError); + } + } + callback(null, presence); + }); + } +} + +export = Backend; +emitter.mixin(Backend); + +Backend.prototype.MIDDLEWARE_ACTIONS = { + // An operation was successfully written to the database. + afterWrite: 'afterWrite', + // An operation is about to be applied to a snapshot before being committed to the database + apply: 'apply', + // An operation was applied to a snapshot; The operation and new snapshot are about to be written to the database. + commit: 'commit', + // A new client connected to the server. + connect: 'connect', + // An operation was loaded from the database + op: 'op', + // A query is about to be sent to the database + query: 'query', + // Snapshot(s) were received from the database and are about to be returned to a client + readSnapshots: 'readSnapshots', + // Received a message from a client + receive: 'receive', + // About to send a non-error reply to a client message. + // WARNING: This gets passed a direct reference to the reply object, so + // be cautious with it. While modifications to the reply message are possible + // by design, changing existing reply properties can cause weird bugs, since + // the rest of ShareDB would be unaware of those changes. + reply: 'reply', + // The server received presence information + receivePresence: 'receivePresence', + // About to send presence information to a client + sendPresence: 'sendPresence', + // An operation is about to be submitted to the database + submit: 'submit' +}; + +Backend.prototype.SNAPSHOT_TYPES = { + // The current snapshot is being fetched (eg through backend.fetch) + current: 'current', + // A specific snapshot is being fetched by version (eg through backend.fetchSnapshot) + byVersion: 'byVersion', + // A specific snapshot is being fetch by timestamp (eg through backend.fetchSnapshotByTimestamp) + byTimestamp: 'byTimestamp' +}; + +function destroyStreams(streams) { + for (var id in streams) { + streams[id].destroy(); + } +} + +function pluckIds(snapshots) { + var ids = []; + for (var i = 0; i < snapshots.length; i++) { + ids.push(snapshots[i].id); + } + return ids; +} + +function filterOpsInPlaceBeforeTimestamp(ops, timestamp) { + for (var i = 0; i < ops.length; i++) { + var op = ops[i]; + var opTimestamp = op.m && op.m.ts; + if (opTimestamp > timestamp) { + ops.length = i; + return; + } + } +} diff --git a/src/client/connection.ts b/src/client/connection.ts new file mode 100644 index 000000000..0785ceb9e --- /dev/null +++ b/src/client/connection.ts @@ -0,0 +1,858 @@ +import Doc = require('./doc'); +import Query = require('./query'); +import Presence = require('./presence/presence'); +import DocPresence = require('./presence/doc-presence'); +import SnapshotVersionRequest = require('./snapshot-request/snapshot-version-request'); +import SnapshotTimestampRequest = require('./snapshot-request/snapshot-timestamp-request'); +import emitter = require('../emitter'); +import ShareDBError = require('../error'); +import { ACTIONS } from '../message-actions'; +import types = require('../types'); +import util = require('../util'); +import logger = require('../logger'); +import DocPresenceEmitter = require('./presence/doc-presence-emitter'); +import protocol = require('../protocol'); + +var ERROR_CODE = ShareDBError.CODES; + +function connectionState(socket) { + if (socket.readyState === 0 || socket.readyState === 1) return 'connecting'; + return 'disconnected'; +} + +export = Connection; + +/** + * Handles communication with the sharejs server and provides queries and + * documents. + * + * We create a connection with a socket object + * connection = new sharejs.Connection(sockset) + * The socket may be any object handling the websocket protocol. See the + * documentation of bindToSocket() for details. We then wait for the connection + * to connect + * connection.on('connected', ...) + * and are finally able to work with shared documents + * connection.get('food', 'steak') // Doc + * + * @param socket @see bindToSocket + */ +class Connection { + collections; + nextQueryId; + nextSnapshotRequestId; + queries; + _presences; + _docPresenceEmitter; + _snapshotRequests; + seq; + _presenceSeq; + id; + agent; + debug; + state; + + constructor(socket) { + emitter.EventEmitter.call(this); + + // Map of collection -> id -> doc object for created documents. + // (created documents MUST BE UNIQUE) + this.collections = Object.create(null); + + // Each query and snapshot request is created with an id that the server uses when it sends us + // info about the request (updates, etc) + this.nextQueryId = 1; + this.nextSnapshotRequestId = 1; + + // Map from query ID -> query object. + this.queries = Object.create(null); + + // Maps from channel -> presence objects + this._presences = Object.create(null); + this._docPresenceEmitter = new DocPresenceEmitter(); + + // Map from snapshot request ID -> snapshot request + this._snapshotRequests = Object.create(null); + + // A unique message number for the given id + this.seq = 1; + + // A unique message number for presence + this._presenceSeq = 1; + + // Equals agent.src on the server + this.id = null; + + // This direct reference from connection to agent is not used internal to + // ShareDB, but it is handy for server-side only user code that may cache + // state on the agent and read it in middleware + this.agent = null; + + this.debug = false; + + this.state = connectionState(socket); + + this.bindToSocket(socket); + } + + /** + * Use socket to communicate with server + * + * Socket is an object that can handle the websocket protocol. This method + * installs the onopen, onclose, onmessage and onerror handlers on the socket to + * handle communication and sends messages by calling socket.send(message). The + * sockets `readyState` property is used to determine the initaial state. + * + * @param socket Handles the websocket protocol + * @param socket.readyState + * @param socket.close + * @param socket.send + * @param socket.onopen + * @param socket.onclose + * @param socket.onmessage + * @param socket.onerror + */ + bindToSocket(socket) { + if (this.socket) { + this.socket.close(); + this.socket.onmessage = null; + this.socket.onopen = null; + this.socket.onerror = null; + this.socket.onclose = null; + } + + this.socket = socket; + + // State of the connection. The corresponding events are emitted when this changes + // + // - 'connecting' The connection is still being established, or we are still + // waiting on the server to send us the initialization message + // - 'connected' The connection is open and we have connected to a server + // and recieved the initialization message + // - 'disconnected' Connection is closed, but it will reconnect automatically + // - 'closed' The connection was closed by the client, and will not reconnect + // - 'stopped' The connection was closed by the server, and will not reconnect + var newState = connectionState(socket); + this._setState(newState); + + // This is a helper variable the document uses to see whether we're + // currently in a 'live' state. It is true if and only if we're connected + this.canSend = false; + + var connection = this; + + socket.onmessage = function(event) { + try { + var data = (typeof event.data === 'string') ? + JSON.parse(event.data) : event.data; + } catch (err) { + logger.warn('Failed to parse message', event); + return; + } + + if (connection.debug) logger.info('RECV', JSON.stringify(data)); + + var request = {data: data}; + connection.emit('receive', request); + if (!request.data) return; + + try { + connection.handleMessage(request.data); + } catch (err) { + util.nextTick(function() { + connection.emit('error', err); + }); + } + }; + + // If socket is already open, do handshake immediately. + if (socket.readyState === 1) { + connection._initializeHandshake(); + } + socket.onopen = function() { + connection._setState('connecting'); + connection._initializeHandshake(); + }; + + socket.onerror = function(err) { + // This isn't the same as a regular error, because it will happen normally + // from time to time. Your connection should probably automatically + // reconnect anyway, but that should be triggered off onclose not onerror. + // (onclose happens when onerror gets called anyway). + connection.emit('connection error', err); + }; + + socket.onclose = function(reason) { + // node-browserchannel reason values: + // 'Closed' - The socket was manually closed by calling socket.close() + // 'Stopped by server' - The server sent the stop message to tell the client not to try connecting + // 'Request failed' - Server didn't respond to request (temporary, usually offline) + // 'Unknown session ID' - Server session for client is missing (temporary, will immediately reestablish) + + if (reason === 'closed' || reason === 'Closed') { + connection._setState('closed', reason); + } else if (reason === 'stopped' || reason === 'Stopped by server') { + connection._setState('stopped', reason); + } else { + connection._setState('disconnected', reason); + } + }; + } + + /** + * @param {object} message + * @param {string} message.a action + */ + handleMessage(message) { + var err = null; + if (message.error) { + err = wrapErrorData(message.error, message); + delete message.error; + } + // Switch on the message action. Most messages are for documents and are + // handled in the doc class. + switch (message.a) { + case ACTIONS.initLegacy: + // Client initialization packet + return this._handleLegacyInit(message); + case ACTIONS.handshake: + return this._handleHandshake(err, message); + case ACTIONS.queryFetch: + var query = this.queries[message.id]; + if (query) query._handleFetch(err, message.data, message.extra); + return; + case ACTIONS.querySubscribe: + var query = this.queries[message.id]; + if (query) query._handleSubscribe(err, message.data, message.extra); + return; + case ACTIONS.queryUnsubscribe: + // Queries are removed immediately on calls to destroy, so we ignore + // replies to query unsubscribes. Perhaps there should be a callback for + // destroy, but this is currently unimplemented + return; + case ACTIONS.queryUpdate: + // Query message. Pass this to the appropriate query object. + var query = this.queries[message.id]; + if (!query) return; + if (err) return query._handleError(err); + if (message.diff) query._handleDiff(message.diff); + if (util.hasOwn(message, 'extra')) query._handleExtra(message.extra); + return; + + case ACTIONS.bulkFetch: + return this._handleBulkMessage(err, message, '_handleFetch'); + case ACTIONS.bulkSubscribe: + case ACTIONS.bulkUnsubscribe: + return this._handleBulkMessage(err, message, '_handleSubscribe'); + + case ACTIONS.snapshotFetch: + case ACTIONS.snapshotFetchByTimestamp: + return this._handleSnapshotFetch(err, message); + + case ACTIONS.fetch: + var doc = this.getExisting(message.c, message.d); + if (doc) doc._handleFetch(err, message.data); + return; + case ACTIONS.subscribe: + case ACTIONS.unsubscribe: + var doc = this.getExisting(message.c, message.d); + if (doc) doc._handleSubscribe(err, message.data); + return; + case ACTIONS.op: + var doc = this.getExisting(message.c, message.d); + if (doc) doc._handleOp(err, message); + return; + case ACTIONS.presence: + return this._handlePresence(err, message); + case ACTIONS.presenceSubscribe: + return this._handlePresenceSubscribe(err, message); + case ACTIONS.presenceUnsubscribe: + return this._handlePresenceUnsubscribe(err, message); + case ACTIONS.presenceRequest: + return this._handlePresenceRequest(err, message); + case ACTIONS.pingPong: + return this._handlePingPong(err); + + default: + logger.warn('Ignoring unrecognized message', message); + } + } + + _handleBulkMessage(err, message, method) { + if (message.data) { + for (var id in message.data) { + var dataForId = message.data[id]; + var doc = this.getExisting(message.c, id); + if (doc) { + if (err) { + doc[method](err); + } else if (dataForId.error) { + // Bulk reply snapshot-specific errorr - see agent.js getMapResult + doc[method](wrapErrorData(dataForId.error)); + } else { + doc[method](null, dataForId); + } + } + } + } else if (Array.isArray(message.b)) { + for (var i = 0; i < message.b.length; i++) { + var id = message.b[i]; + var doc = this.getExisting(message.c, id); + if (doc) doc[method](err); + } + } else if (message.b) { + for (var id in message.b) { + var doc = this.getExisting(message.c, id); + if (doc) doc[method](err); + } + } else { + logger.error('Invalid bulk message', message); + } + } + + _reset() { + this.agent = null; + } + + // Set the connection's state. The connection is basically a state machine. + _setState(newState, reason) { + if (this.state === newState) return; + + // I made a state diagram. The only invalid transitions are getting to + // 'connecting' from anywhere other than 'disconnected' and getting to + // 'connected' from anywhere other than 'connecting'. + if ( + ( + newState === 'connecting' && + this.state !== 'disconnected' && + this.state !== 'stopped' && + this.state !== 'closed' + ) || ( + newState === 'connected' && + this.state !== 'connecting' + ) + ) { + var err = new ShareDBError( + ERROR_CODE.ERR_CONNECTION_STATE_TRANSITION_INVALID, + 'Cannot transition directly from ' + this.state + ' to ' + newState + ); + return this.emit('error', err); + } + + this.state = newState; + this.canSend = (newState === 'connected'); + + if ( + newState === 'disconnected' || + newState === 'stopped' || + newState === 'closed' + ) { + this._reset(); + } + + // Group subscribes together to help server make more efficient calls + this.startBulk(); + // Emit the event to all queries + for (var id in this.queries) { + var query = this.queries[id]; + query._onConnectionStateChanged(); + } + // Emit the event to all documents + for (var collection in this.collections) { + var docs = this.collections[collection]; + for (var id in docs) { + docs[id]._onConnectionStateChanged(); + } + } + // Emit the event to all Presences + for (var channel in this._presences) { + this._presences[channel]._onConnectionStateChanged(); + } + // Emit the event to all snapshots + for (var id in this._snapshotRequests) { + var snapshotRequest = this._snapshotRequests[id]; + snapshotRequest._onConnectionStateChanged(); + } + this.endBulk(); + + this.emit(newState, reason); + this.emit('state', newState, reason); + } + + startBulk() { + if (!this.bulk) this.bulk = Object.create(null); + } + + endBulk() { + if (this.bulk) { + for (var collection in this.bulk) { + var actions = this.bulk[collection]; + this._sendBulk('f', collection, actions.f); + this._sendBulk('s', collection, actions.s); + this._sendBulk('u', collection, actions.u); + } + } + this.bulk = null; + } + + _sendBulk(action, collection, values) { + if (!values) return; + var ids = []; + var versions = Object.create(null); + var versionsCount = 0; + var versionId; + for (var id in values) { + var value = values[id]; + if (value == null) { + ids.push(id); + } else { + versions[id] = value; + versionId = id; + versionsCount++; + } + } + if (ids.length === 1) { + var id = ids[0]; + this.send({a: action, c: collection, d: id}); + } else if (ids.length) { + this.send({a: 'b' + action, c: collection, b: ids}); + } + if (versionsCount === 1) { + var version = versions[versionId]; + this.send({a: action, c: collection, d: versionId, v: version}); + } else if (versionsCount) { + this.send({a: 'b' + action, c: collection, b: versions}); + } + } + + _sendActions(action, doc, version) { + // Ensure the doc is registered so that it receives the reply message + this._addDoc(doc); + if (this.bulk) { + // Bulk subscribe + var actions = this.bulk[doc.collection] || (this.bulk[doc.collection] = Object.create(null)); + var versions = actions[action] || (actions[action] = Object.create(null)); + var isDuplicate = util.hasOwn(versions, doc.id); + versions[doc.id] = version; + return isDuplicate; + } else { + // Send single doc subscribe message + var message = {a: action, c: doc.collection, d: doc.id, v: version}; + this.send(message); + } + } + + sendFetch(doc) { + return this._sendActions(ACTIONS.fetch, doc, doc.version); + } + + sendSubscribe(doc) { + return this._sendActions(ACTIONS.subscribe, doc, doc.version); + } + + sendUnsubscribe(doc) { + return this._sendActions(ACTIONS.unsubscribe, doc); + } + + sendOp(doc, op) { + // Ensure the doc is registered so that it receives the reply message + this._addDoc(doc); + var message = { + a: ACTIONS.op, + c: doc.collection, + d: doc.id, + v: doc.version, + src: op.src, + seq: op.seq, + x: {} + }; + if ('op' in op) message.op = op.op; + if (op.create) message.create = op.create; + if (op.del) message.del = op.del; + if (doc.submitSource) message.x.source = op.source; + this.send(message); + } + + /** + * Sends a message down the socket + */ + send(message) { + if (this.debug) logger.info('SEND', JSON.stringify(message)); + + this.emit('send', message); + this.socket.send(JSON.stringify(message)); + } + + ping() { + if (!this.canSend) { + throw new ShareDBError( + ERROR_CODE.ERR_CANNOT_PING_OFFLINE, + 'Socket must be CONNECTED to ping' + ); + } + + var message = { + a: ACTIONS.pingPong + }; + this.send(message); + } + + /** + * Closes the socket and emits 'closed' + */ + close() { + this.socket.close(); + } + + getExisting(collection, id) { + if (this.collections[collection]) return this.collections[collection][id]; + } + + /** + * Get or create a document. + * + * @param collection + * @param id + * @return {Doc} + */ + get(collection, id) { + var docs = this.collections[collection] || + (this.collections[collection] = Object.create(null)); + + var doc = docs[id]; + if (!doc) { + doc = docs[id] = new Doc(this, collection, id); + this.emit('doc', doc); + } + + doc._wantsDestroy = false; + return doc; + } + + /** + * Remove document from this.collections + * + * @private + */ + _destroyDoc(doc) { + if (!doc._wantsDestroy) return; + util.digAndRemove(this.collections, doc.collection, doc.id); + doc.emit('destroy'); + } + + _addDoc(doc) { + var docs = this.collections[doc.collection]; + if (!docs) { + docs = this.collections[doc.collection] = Object.create(null); + } + if (docs[doc.id] !== doc) { + docs[doc.id] = doc; + } + } + + // Helper for createFetchQuery and createSubscribeQuery, below. + _createQuery(action, collection, q, options, callback) { + var id = this.nextQueryId++; + var query = new Query(action, this, id, collection, q, options, callback); + this.queries[id] = query; + query.send(); + return query; + } + + // Internal function. Use query.destroy() to remove queries. + _destroyQuery(query) { + delete this.queries[query.id]; + } + + // The query options object can contain the following fields: + // + // db: Name of the db for the query. You can attach extraDbs to ShareDB and + // pick which one the query should hit using this parameter. + + // Create a fetch query. Fetch queries are only issued once, returning the + // results directly into the callback. + // + // The callback should have the signature function(error, results, extra) + // where results is a list of Doc objects. + createFetchQuery(collection, q, options, callback) { + return this._createQuery(ACTIONS.queryFetch, collection, q, options, callback); + } + + // Create a subscribe query. Subscribe queries return with the initial data + // through the callback, then update themselves whenever the query result set + // changes via their own event emitter. + // + // If present, the callback should have the signature function(error, results, extra) + // where results is a list of Doc objects. + createSubscribeQuery(collection, q, options, callback) { + return this._createQuery(ACTIONS.querySubscribe, collection, q, options, callback); + } + + hasPending() { + return !!( + this._firstDoc(hasPending) || + this._firstQuery(hasPending) || + this._firstSnapshotRequest() + ); + } + + hasWritePending() { + return !!this._firstDoc(hasWritePending); + } + + whenNothingPending(callback) { + var doc = this._firstDoc(hasPending); + if (doc) { + // If a document is found with a pending operation, wait for it to emit + // that nothing is pending anymore, and then recheck all documents again. + // We have to recheck all documents, just in case another mutation has + // been made in the meantime as a result of an event callback + doc.once('nothing pending', this._nothingPendingRetry(callback)); + return; + } + var query = this._firstQuery(hasPending); + if (query) { + query.once('ready', this._nothingPendingRetry(callback)); + return; + } + var snapshotRequest = this._firstSnapshotRequest(); + if (snapshotRequest) { + snapshotRequest.once('ready', this._nothingPendingRetry(callback)); + return; + } + // Call back when no pending operations + util.nextTick(callback); + } + + _nothingPendingRetry(callback) { + var connection = this; + return function() { + util.nextTick(function() { + connection.whenNothingPending(callback); + }); + }; + } + + _firstDoc(fn) { + for (var collection in this.collections) { + var docs = this.collections[collection]; + for (var id in docs) { + var doc = docs[id]; + if (fn(doc)) { + return doc; + } + } + } + } + + _firstQuery(fn) { + for (var id in this.queries) { + var query = this.queries[id]; + if (fn(query)) { + return query; + } + } + } + + _firstSnapshotRequest() { + for (var id in this._snapshotRequests) { + return this._snapshotRequests[id]; + } + } + + /** + * Fetch a read-only snapshot at a given version + * + * @param collection - the collection name of the snapshot + * @param id - the ID of the snapshot + * @param version (optional) - the version number to fetch. If null, the latest version is fetched. + * @param callback - (error, snapshot) => void, where snapshot takes the following schema: + * + * { + * id: string; // ID of the snapshot + * v: number; // version number of the snapshot + * type: string; // the OT type of the snapshot, or null if it doesn't exist or is deleted + * data: any; // the snapshot + * } + * + */ + fetchSnapshot(collection, id, version, callback) { + if (typeof version === 'function') { + callback = version; + version = null; + } + + var requestId = this.nextSnapshotRequestId++; + var snapshotRequest = new SnapshotVersionRequest(this, requestId, collection, id, version, callback); + this._snapshotRequests[snapshotRequest.requestId] = snapshotRequest; + snapshotRequest.send(); + } + + /** + * Fetch a read-only snapshot at a given timestamp + * + * @param collection - the collection name of the snapshot + * @param id - the ID of the snapshot + * @param timestamp (optional) - the timestamp to fetch. If null, the latest version is fetched. + * @param callback - (error, snapshot) => void, where snapshot takes the following schema: + * + * { + * id: string; // ID of the snapshot + * v: number; // version number of the snapshot + * type: string; // the OT type of the snapshot, or null if it doesn't exist or is deleted + * data: any; // the snapshot + * } + * + */ + fetchSnapshotByTimestamp(collection, id, timestamp, callback) { + if (typeof timestamp === 'function') { + callback = timestamp; + timestamp = null; + } + + var requestId = this.nextSnapshotRequestId++; + var snapshotRequest = new SnapshotTimestampRequest(this, requestId, collection, id, timestamp, callback); + this._snapshotRequests[snapshotRequest.requestId] = snapshotRequest; + snapshotRequest.send(); + } + + _handleSnapshotFetch(error, message) { + var snapshotRequest = this._snapshotRequests[message.id]; + if (!snapshotRequest) return; + delete this._snapshotRequests[message.id]; + snapshotRequest._handleResponse(error, message); + } + + _handleLegacyInit(message) { + // If the protocol is at least 1.1, we want to use the + // new handshake protocol. Let's send a handshake initialize, because + // we now know the server is ready. If we've already sent it, we'll + // just ignore the response anyway. + if (protocol.checkAtLeast(message, '1.1')) return this._initializeHandshake(); + this._initialize(message); + } + + _initializeHandshake() { + this.send({ + a: ACTIONS.handshake, + id: this.id, + protocol: protocol.major, + protocolMinor: protocol.minor + }); + } + + _handleHandshake(error, message) { + if (error) return this.emit('error', error); + this._initialize(message); + } + + _handlePingPong(error) { + if (error) return this.emit('error', error); + this.emit('pong'); + } + + _initialize(message) { + if (this.state !== 'connecting') return; + + if (message.protocol !== protocol.major) { + return this.emit('error', new ShareDBError( + ERROR_CODE.ERR_PROTOCOL_VERSION_NOT_SUPPORTED, + 'Unsupported protocol version: ' + message.protocol + )); + } + if (types.map[message.type] !== types.defaultType) { + return this.emit('error', new ShareDBError( + ERROR_CODE.ERR_DEFAULT_TYPE_MISMATCH, + message.type + ' does not match the server default type' + )); + } + if (typeof message.id !== 'string') { + return this.emit('error', new ShareDBError( + ERROR_CODE.ERR_CLIENT_ID_BADLY_FORMED, + 'Client id must be a string' + )); + } + this.id = message.id; + + this._setState('connected'); + } + + getPresence(channel) { + var connection = this; + var presence = util.digOrCreate(this._presences, channel, function() { + return new Presence(connection, channel); + }); + presence._wantsDestroy = false; + return presence; + } + + getDocPresence(collection, id) { + var channel = DocPresence.channel(collection, id); + var connection = this; + var presence = util.digOrCreate(this._presences, channel, function() { + return new DocPresence(connection, collection, id); + }); + presence._wantsDestroy = false; + return presence; + } + + _sendPresenceAction(action, seq, presence) { + // Ensure the presence is registered so that it receives the reply message + this._addPresence(presence); + var message = {a: action, ch: presence.channel, seq: seq}; + this.send(message); + return message.seq; + } + + _addPresence(presence) { + util.digOrCreate(this._presences, presence.channel, function() { + return presence; + }); + } + + _requestRemotePresence(channel) { + this.send({a: ACTIONS.presenceRequest, ch: channel}); + } + + _handlePresenceSubscribe(error, message) { + var presence = util.dig(this._presences, message.ch); + if (presence) presence._handleSubscribe(error, message.seq); + } + + _handlePresenceUnsubscribe(error, message) { + var presence = util.dig(this._presences, message.ch); + if (presence) presence._handleUnsubscribe(error, message.seq); + } + + _handlePresence(error, message) { + var presence = util.dig(this._presences, message.ch); + if (presence) presence._receiveUpdate(error, message); + } + + _handlePresenceRequest(error, message) { + var presence = util.dig(this._presences, message.ch); + if (presence) presence._broadcastAllLocalPresence(error, message); + } +} + +emitter.mixin(Connection); + + +function wrapErrorData(errorData, fullMessage) { + // wrap in Error object so can be passed through event emitters + var err = new Error(errorData.message); + err.code = errorData.code; + if (fullMessage) { + // Add the message data to the error object for more context + err.data = fullMessage; + } + return err; +} + +function hasPending(object) { + return object.hasPending(); +} + +function hasWritePending(object) { + return object.hasWritePending(); +} diff --git a/src/client/doc.ts b/src/client/doc.ts new file mode 100644 index 000000000..c474a352f --- /dev/null +++ b/src/client/doc.ts @@ -0,0 +1,1141 @@ +import emitter = require('../emitter'); +import logger = require('../logger'); +import ShareDBError = require('../error'); +import types = require('../types'); +import util = require('../util'); +var clone = util.clone; +import deepEqual = require('fast-deep-equal'); +import { ACTIONS } from '../message-actions'; + +var ERROR_CODE = ShareDBError.CODES; + +export = Doc; + +/** + * A Doc is a client's view on a sharejs document. + * + * It is is uniquely identified by its `id` and `collection`. Documents + * should not be created directly. Create them with connection.get() + * + * + * Subscriptions + * ------------- + * + * We can subscribe a document to stay in sync with the server. + * doc.subscribe(function(error) { + * doc.subscribed // = true + * }) + * The server now sends us all changes concerning this document and these are + * applied to our data. If the subscription was successful the initial + * data and version sent by the server are loaded into the document. + * + * To stop listening to the changes we call `doc.unsubscribe()`. + * + * If we just want to load the data but not stay up-to-date, we call + * doc.fetch(function(error) { + * doc.data // sent by server + * }) + * + * + * Events + * ------ + * + * You can use doc.on(eventName, callback) to subscribe to the following events: + * - `before op (op, source)` Fired before a partial operation is applied to the data. + * It may be used to read the old data just before applying an operation + * - `op (op, source)` Fired after every partial operation with this operation as the + * first argument + * - `create (source)` The document was created. That means its type was + * set and it has some initial data. + * - `del (data, source)` Fired after the document is deleted, that is + * the data is null. It is passed the data before deletion as an + * argument + * - `load ()` Fired when a new snapshot is ingested from a fetch, subscribe, or query + */ +class Doc { + connection; + collection; + id; + version; + type; + data; + inflightFetch; + inflightSubscribe; + pendingFetch; + pendingSubscribe; + _isInHardRollback; + subscribed; + wantSubscribe; + _wantsDestroy; + inflightOp; + pendingOps; + applyStack; + preventCompose; + submitSource; + paused; + _dataStateVersion; + + constructor(connection, collection, id) { + emitter.EventEmitter.call(this); + + this.connection = connection; + + this.collection = collection; + this.id = id; + + this.version = null; + // The OT type of this document. An uncreated document has type `null` + this.type = null; + this.data = undefined; + + // Array of callbacks or nulls as placeholders + this.inflightFetch = []; + this.inflightSubscribe = null; + this.pendingFetch = []; + this.pendingSubscribe = []; + + this._isInHardRollback = false; + + // Whether we think we are subscribed on the server. Synchronously set to + // false on calls to unsubscribe and disconnect. Should never be true when + // this.wantSubscribe is false + this.subscribed = false; + // Whether to re-establish the subscription on reconnect + this.wantSubscribe = false; + + this._wantsDestroy = false; + + // The op that is currently roundtripping to the server, or null. + // + // When the connection reconnects, the inflight op is resubmitted. + // + // This has the same format as an entry in pendingOps + this.inflightOp = null; + + // All ops that are waiting for the server to acknowledge this.inflightOp + // This used to just be a single operation, but creates & deletes can't be + // composed with regular operations. + // + // This is a list of {[create:{...}], [del:true], [op:...], callbacks:[...]} + this.pendingOps = []; + + // The applyStack enables us to track any ops submitted while we are + // applying an op incrementally. This value is an array when we are + // performing an incremental apply and null otherwise. When it is an array, + // all submitted ops should be pushed onto it. The `_otApply` method will + // reset it back to null when all incremental apply loops are complete. + this.applyStack = null; + + // Disable the default behavior of composing submitted ops. This is read at + // the time of op submit, so it may be toggled on before submitting a + // specifc op and toggled off afterward + this.preventCompose = false; + + // If set to true, the source will be submitted over the connection. This + // will also have the side-effect of only composing ops whose sources are + // equal + this.submitSource = false; + + // Prevent own ops being submitted to the server. If subscribed, remote + // ops are still received. Should be toggled through the pause() and + // resume() methods to correctly flush on resume. + this.paused = false; + + // Internal counter that gets incremented every time doc.data is updated. + // Used as a cheap way to check if doc.data has changed. + this._dataStateVersion = 0; + } + + destroy(callback) { + this._wantsDestroy = true; + var doc = this; + doc.whenNothingPending(function() { + if (doc.wantSubscribe) { + doc.unsubscribe(function(err) { + if (err) { + if (callback) return callback(err); + return doc.emit('error', err); + } + doc.connection._destroyDoc(doc); + if (callback) callback(); + }); + } else { + doc.connection._destroyDoc(doc); + if (callback) callback(); + } + }); + } + + // ****** Manipulating the document data, version and type. + + // Set the document's type, and associated properties. Most of the logic in + // this function exists to update the document based on any added & removed API + // methods. + // + // @param newType OT type provided by the ottypes library or its name or uri + _setType(newType) { + if (typeof newType === 'string') { + newType = types.map[newType]; + } + + if (newType) { + this.type = newType; + } else if (newType === null) { + this.type = newType; + // If we removed the type from the object, also remove its data + this._setData(undefined); + } else { + var err = new ShareDBError(ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, 'Missing type ' + newType); + return this.emit('error', err); + } + } + + _setData(data) { + this.data = data; + this._dataStateVersion++; + } + + // Ingest snapshot data. This data must include a version, snapshot and type. + // This is used both to ingest data that was exported with a webpage and data + // that was received from the server during a fetch. + // + // @param snapshot.v version + // @param snapshot.data + // @param snapshot.type + // @param callback + ingestSnapshot(snapshot, callback) { + if (!snapshot) return callback && callback(); + + if (typeof snapshot.v !== 'number') { + var err = new ShareDBError( + ERROR_CODE.ERR_INGESTED_SNAPSHOT_HAS_NO_VERSION, + 'Missing version in ingested snapshot. ' + this.collection + '.' + this.id + ); + if (callback) return callback(err); + return this.emit('error', err); + } + + // If the doc is already created or there are ops pending, we cannot use the + // ingested snapshot and need ops in order to update the document + if (this.type || this.hasWritePending()) { + // The version should only be null on a created document when it was + // created locally without fetching + if (this.version == null) { + if (this.hasWritePending()) { + // If we have pending ops and we get a snapshot for a locally created + // document, we have to wait for the pending ops to complete, because + // we don't know what version to fetch ops from. It is possible that + // the snapshot came from our local op, but it is also possible that + // the doc was created remotely (which would conflict and be an error) + return callback && this.once('no write pending', callback); + } + // Otherwise, we've encounted an error state + var err = new ShareDBError( + ERROR_CODE.ERR_DOC_MISSING_VERSION, + 'Cannot ingest snapshot in doc with null version. ' + this.collection + '.' + this.id + ); + if (callback) return callback(err); + return this.emit('error', err); + } + // If we got a snapshot for a version further along than the document is + // currently, issue a fetch to get the latest ops and catch us up + if (snapshot.v > this.version) return this.fetch(callback); + return callback && callback(); + } + + // Ignore the snapshot if we are already at a newer version. Under no + // circumstance should we ever set the current version backward + if (this.version > snapshot.v) return callback && callback(); + + this.version = snapshot.v; + var type = (snapshot.type === undefined) ? types.defaultType : snapshot.type; + this._setType(type); + this._setData( + (this.type && this.type.deserialize) ? + this.type.deserialize(snapshot.data) : + snapshot.data + ); + this.emit('load'); + callback && callback(); + } + + whenNothingPending(callback) { + var doc = this; + util.nextTick(function() { + if (doc.hasPending()) { + doc.once('nothing pending', callback); + return; + } + callback(); + }); + } + + hasPending() { + return !!( + this.inflightOp || + this.pendingOps.length || + this.inflightFetch.length || + this.inflightSubscribe || + this.pendingFetch.length || + this.pendingSubscribe.length + ); + } + + hasWritePending() { + return !!(this.inflightOp || this.pendingOps.length); + } + + _emitNothingPending() { + if (this.hasWritePending()) return; + this.emit('no write pending'); + if (this.hasPending()) return; + this.emit('nothing pending'); + } + + // **** Helpers for network messages + + _emitResponseError(err, callback) { + if (err && err.code === ERROR_CODE.ERR_SNAPSHOT_READ_SILENT_REJECTION) { + this.wantSubscribe = false; + if (callback) { + callback(); + } + this._emitNothingPending(); + return; + } + if (callback) { + callback(err); + this._emitNothingPending(); + return; + } + this._emitNothingPending(); + this.emit('error', err); + } + + _handleFetch(error, snapshot) { + var callbacks = this.pendingFetch; + this.pendingFetch = []; + var callback = this.inflightFetch.shift(); + if (callback) callbacks.unshift(callback); + if (callbacks.length) { + callback = function(error) { + util.callEach(callbacks, error); + }; + } + if (error) return this._emitResponseError(error, callback); + this.ingestSnapshot(snapshot, callback); + this._emitNothingPending(); + } + + _handleSubscribe(error, snapshot) { + var request = this.inflightSubscribe; + this.inflightSubscribe = null; + var callbacks = this.pendingFetch; + this.pendingFetch = []; + if (request.callback) callbacks.push(request.callback); + var callback; + if (callbacks.length) { + callback = function(error) { + util.callEach(callbacks, error); + }; + } + if (error) return this._emitResponseError(error, callback); + this.subscribed = request.wantSubscribe; + if (this.subscribed) this.ingestSnapshot(snapshot, callback); + else if (callback) callback(); + this._emitNothingPending(); + this._flushSubscribe(); + } + + _handleOp(err, message) { + if (err) { + if (err.code === ERROR_CODE.ERR_NO_OP && message.seq === this.inflightOp.seq) { + // Our op was a no-op, either because we submitted a no-op, or - more + // likely - because our op was transformed into a no-op by the server + // because of a similar remote op. In this case, the server has avoided + // committing the op to the database, and we should just clear the in-flight + // op and call the callbacks. However, let's first catch ourselves up to + // the remote, so that we're in a nice consistent state + return this.fetch(this._clearInflightOp.bind(this)); + } + if (this.inflightOp) { + return this._rollback(err); + } + return this.emit('error', err); + } + + if (this.inflightOp && + message.src === this.inflightOp.src && + message.seq === this.inflightOp.seq) { + // The op has already been applied locally. Just update the version + // and pending state appropriately + this._opAcknowledged(message); + return; + } + + if (this.version == null || message.v > this.version) { + // This will happen in normal operation if we become subscribed to a + // new document via a query. It can also happen if we get an op for + // a future version beyond the version we are expecting next. This + // could happen if the server doesn't publish an op for whatever reason + // or because of a race condition. In any case, we can send a fetch + // command to catch back up. + // + // Fetch only sends a new fetch command if no fetches are inflight, which + // will act as a natural debouncing so we don't send multiple fetch + // requests for many ops received at once. + this.fetch(); + return; + } + + if (message.v < this.version) { + // We can safely ignore the old (duplicate) operation. + return; + } + + if (this.inflightOp) { + var transformErr = transformX(this.inflightOp, message); + if (transformErr) return this._hardRollback(transformErr); + } + + for (var i = 0; i < this.pendingOps.length; i++) { + var transformErr = transformX(this.pendingOps[i], message); + if (transformErr) return this._hardRollback(transformErr); + } + + this.version++; + try { + this._otApply(message, false); + } catch (error) { + return this._hardRollback(error); + } + } + + // Called whenever (you guessed it!) the connection state changes. This will + // happen when we get disconnected & reconnect. + _onConnectionStateChanged() { + if (this.connection.canSend) { + this.flush(); + this._resubscribe(); + } else { + if (this.inflightOp) { + this.pendingOps.unshift(this.inflightOp); + this.inflightOp = null; + } + this.subscribed = false; + if (this.inflightSubscribe) { + if (this.inflightSubscribe.wantSubscribe) { + this.pendingSubscribe.unshift(this.inflightSubscribe); + this.inflightSubscribe = null; + } else { + this._handleSubscribe(); + } + } + if (this.inflightFetch.length) { + this.pendingFetch = this.pendingFetch.concat(this.inflightFetch); + this.inflightFetch.length = 0; + } + } + } + + _resubscribe() { + if (!this.pendingSubscribe.length && this.wantSubscribe) { + return this.subscribe(); + } + var willFetch = this.pendingSubscribe.some(function(request) { + return request.wantSubscribe; + }); + if (!willFetch && this.pendingFetch.length) this.fetch(); + this._flushSubscribe(); + } + + // Request the current document snapshot or ops that bring us up to date + fetch(callback) { + this._fetch({}, callback); + } + + _fetch(options, callback) { + this.pendingFetch.push(callback); + var shouldSend = this.connection.canSend && ( + options.force || !this.inflightFetch.length + ); + if (!shouldSend) return; + this.inflightFetch.push(this.pendingFetch.shift()); + this.connection.sendFetch(this); + } + + // Fetch the initial document and keep receiving updates + subscribe(callback) { + var wantSubscribe = true; + this._queueSubscribe(wantSubscribe, callback); + } + + // Unsubscribe. The data will stay around in local memory, but we'll stop + // receiving updates + unsubscribe(callback) { + var wantSubscribe = false; + this._queueSubscribe(wantSubscribe, callback); + } + + _queueSubscribe(wantSubscribe, callback) { + var lastRequest = this.pendingSubscribe[this.pendingSubscribe.length - 1] || this.inflightSubscribe; + var isDuplicateRequest = lastRequest && lastRequest.wantSubscribe === wantSubscribe; + if (isDuplicateRequest) { + lastRequest.callback = combineCallbacks([lastRequest.callback, callback]); + return; + } + this.pendingSubscribe.push({ + wantSubscribe: !!wantSubscribe, + callback: callback + }); + this._flushSubscribe(); + } + + _flushSubscribe() { + if (this.inflightSubscribe || !this.pendingSubscribe.length) return; + + if (this.connection.canSend) { + this.inflightSubscribe = this.pendingSubscribe.shift(); + this.wantSubscribe = this.inflightSubscribe.wantSubscribe; + if (this.wantSubscribe) { + this.connection.sendSubscribe(this); + } else { + // Be conservative about our subscription state. We'll be unsubscribed + // some time between sending this request, and receiving the callback, + // so let's just set ourselves to unsubscribed now. + this.subscribed = false; + this.connection.sendUnsubscribe(this); + } + + return; + } + + // If we're offline, then we're already unsubscribed. Therefore, call back + // the next request immediately if it's an unsubscribe request. + if (!this.pendingSubscribe[0].wantSubscribe) { + this.inflightSubscribe = this.pendingSubscribe.shift(); + var doc = this; + util.nextTick(function() { + doc._handleSubscribe(); + }); + } + } + + // Operations // + + // Send the next pending op to the server, if we can. + // + // Only one operation can be in-flight at a time. If an operation is already on + // its way, or we're not currently connected, this method does nothing. + flush() { + // Ignore if we can't send or we are already sending an op + if (!this.connection.canSend || this.inflightOp) return; + + // Send first pending op unless paused + if (!this.paused && this.pendingOps.length) { + this._sendOp(); + } + } + + /** + * Applies the operation to the snapshot + * + * If the operation is create or delete it emits `create` or `del`. Then the + * operation is applied to the snapshot and `op` and `after op` are emitted. + * If the type supports incremental updates and `this.incremental` is true we + * fire `op` after every small operation. + * + * This is the only function to fire the above mentioned events. + * + * @private + */ + _otApply(op, source) { + if ('op' in op) { + if (!this.type) { + // Throw here, because all usage of _otApply should be wrapped with a try/catch + throw new ShareDBError( + ERROR_CODE.ERR_DOC_DOES_NOT_EXIST, + 'Cannot apply op to uncreated document. ' + this.collection + '.' + this.id + ); + } + + // NB: If we need to add another argument to this event, we should consider + // the fact that the 'op' event has op.src as its 3rd argument + this.emit('before op batch', op.op, source); + + // Iteratively apply multi-component remote operations and rollback ops + // (source === false) for the default JSON0 OT type. It could use + // type.shatter(), but since this code is so specific to use cases for the + // JSON0 type and ShareDB explicitly bundles the default type, we might as + // well write it this way and save needing to iterate through the op + // components twice. + // + // Ideally, we would not need this extra complexity. However, it is + // helpful for implementing bindings that update DOM nodes and other + // stateful objects by translating op events directly into corresponding + // mutations. Such bindings are most easily written as responding to + // individual op components one at a time in order, and it is important + // that the snapshot only include updates from the particular op component + // at the time of emission. Eliminating this would require rethinking how + // such external bindings are implemented. + if (!source && this.type === types.defaultType && op.op.length > 1) { + if (!this.applyStack) this.applyStack = []; + var stackLength = this.applyStack.length; + for (var i = 0; i < op.op.length; i++) { + var component = op.op[i]; + var componentOp = {op: [component]}; + // Apply the individual op component + this.emit('before op', componentOp.op, source, op.src); + // Transform componentOp against any ops that have been submitted + // sychronously inside of an op event handler since we began apply of + // our operation + for (var j = stackLength; j < this.applyStack.length; j++) { + var transformErr = transformX(this.applyStack[j], componentOp); + if (transformErr) return this._hardRollback(transformErr); + } + this._setData(this.type.apply(this.data, componentOp.op)); + this.emit('op', componentOp.op, source, op.src); + } + this.emit('op batch', op.op, source); + // Pop whatever was submitted since we started applying this op + this._popApplyStack(stackLength); + return; + } + + // The 'before op' event enables clients to pull any necessary data out of + // the snapshot before it gets changed + this.emit('before op', op.op, source, op.src); + // Apply the operation to the local data, mutating it in place + this._setData(this.type.apply(this.data, op.op)); + // Emit an 'op' event once the local data includes the changes from the + // op. For locally submitted ops, this will be synchronously with + // submission and before the server or other clients have received the op. + // For ops from other clients, this will be after the op has been + // committed to the database and published + this.emit('op', op.op, source, op.src); + this.emit('op batch', op.op, source); + return; + } + + if (op.create) { + this._setType(op.create.type); + if (this.type.deserialize) { + if (this.type.createDeserialized) { + this._setData(this.type.createDeserialized(op.create.data)); + } else { + this._setData(this.type.deserialize(this.type.create(op.create.data))); + } + } else { + this._setData(this.type.create(op.create.data)); + } + this.emit('create', source); + return; + } + + if (op.del) { + var oldData = this.data; + this._setType(null); + this.emit('del', oldData, source); + return; + } + } + + // ***** Sending operations + + // Actually send op to the server. + _sendOp() { + if (!this.connection.canSend) return; + var src = this.connection.id; + + // When there is no inflightOp, send the first item in pendingOps. If + // there is inflightOp, try sending it again + if (!this.inflightOp) { + // Send first pending op + this.inflightOp = this.pendingOps.shift(); + } + var op = this.inflightOp; + if (!op) { + var err = new ShareDBError(ERROR_CODE.ERR_INFLIGHT_OP_MISSING, 'No op to send on call to _sendOp'); + return this.emit('error', err); + } + + // Track data for retrying ops + op.sentAt = Date.now(); + op.retries = (op.retries == null) ? 0 : op.retries + 1; + + // The src + seq number is a unique ID representing this operation. This tuple + // is used on the server to detect when ops have been sent multiple times and + // on the client to match acknowledgement of an op back to the inflightOp. + // Note that the src could be different from this.connection.id after a + // reconnect, since an op may still be pending after the reconnection and + // this.connection.id will change. In case an op is sent multiple times, we + // also need to be careful not to override the original seq value. + if (op.seq == null) { + if (this.connection.seq >= util.MAX_SAFE_INTEGER) { + return this.emit('error', new ShareDBError( + ERROR_CODE.ERR_CONNECTION_SEQ_INTEGER_OVERFLOW, + 'Connection seq has exceeded the max safe integer, maybe from being open for too long' + )); + } + + op.seq = this.connection.seq++; + } + + this.connection.sendOp(this, op); + + // src isn't needed on the first try, since the server session will have the + // same id, but it must be set on the inflightOp in case it is sent again + // after a reconnect and the connection's id has changed by then + if (op.src == null) op.src = src; + } + + // Queues the operation for submission to the server and applies it locally. + // + // Internal method called to do the actual work for submit(), create() and del(). + // @private + // + // @param op + // @param [op.op] + // @param [op.del] + // @param [op.create] + // @param [callback] called when operation is submitted + _submit(op, source, callback) { + // Locally submitted ops must always have a truthy source + if (!source) source = true; + + // The op contains either op, create, delete, or none of the above (a no-op). + if ('op' in op) { + if (!this.type) { + if (this._isInHardRollback) { + var err = new ShareDBError( + ERROR_CODE.ERR_DOC_IN_HARD_ROLLBACK, + 'Cannot submit op. Document is performing hard rollback. ' + this.collection + '.' + this.id + ); + } else { + var err = new ShareDBError( + ERROR_CODE.ERR_DOC_DOES_NOT_EXIST, + 'Cannot submit op. Document has not been created. ' + this.collection + '.' + this.id + ); + } + + if (callback) return callback(err); + return this.emit('error', err); + } + // Try to normalize the op. This removes trailing skip:0's and things like that. + if (this.type.normalize) op.op = this.type.normalize(op.op); + } + + try { + this._pushOp(op, source, callback); + this._otApply(op, source); + } catch (error) { + return this._hardRollback(error); + } + + // The call to flush is delayed so if submit() is called multiple times + // synchronously, all the ops are combined before being sent to the server. + var doc = this; + util.nextTick(function() { + doc.flush(); + }); + } + + _pushOp(op, source, callback) { + op.source = source; + if (this.applyStack) { + // If we are in the process of incrementally applying an operation, don't + // compose the op and push it onto the applyStack so it can be transformed + // against other components from the op or ops being applied + this.applyStack.push(op); + } else { + // If the type supports composes, try to compose the operation onto the + // end of the last pending operation. + var composed = this._tryCompose(op); + if (composed) { + composed.callbacks.push(callback); + return; + } + } + // Push on to the pendingOps queue of ops to submit if we didn't compose + op.type = this.type; + op.callbacks = [callback]; + this.pendingOps.push(op); + } + + _popApplyStack(to) { + if (to > 0) { + this.applyStack.length = to; + return; + } + // Once we have completed the outermost apply loop, reset to null and no + // longer add ops to the applyStack as they are submitted + var op = this.applyStack[0]; + this.applyStack = null; + if (!op) return; + // Compose the ops added since the beginning of the apply stack, since we + // had to skip compose when they were originally pushed + var i = this.pendingOps.indexOf(op); + if (i === -1) return; + var ops = this.pendingOps.splice(i); + for (var i = 0; i < ops.length; i++) { + var op = ops[i]; + var composed = this._tryCompose(op); + if (composed) { + composed.callbacks = composed.callbacks.concat(op.callbacks); + } else { + this.pendingOps.push(op); + } + } + } + + // Try to compose a submitted op into the last pending op. Returns the + // composed op if it succeeds, undefined otherwise + _tryCompose(op) { + if (this.preventCompose) return; + + // We can only compose into the last pending op. Inflight ops have already + // been sent to the server, so we can't modify them + var last = this.pendingOps[this.pendingOps.length - 1]; + if (!last || last.sentAt) return; + + // If we're submitting the op source, we can only combine ops that have + // a matching source + if (this.submitSource && !deepEqual(op.source, last.source)) return; + + // Compose an op into a create by applying it. This effectively makes the op + // invisible, as if the document were created including the op originally + if (last.create && 'op' in op) { + last.create.data = this.type.apply(last.create.data, op.op); + return last; + } + + // Compose two ops into a single op if supported by the type. Types that + // support compose must be able to compose any two ops together + if ('op' in last && 'op' in op && this.type.compose) { + last.op = this.type.compose(last.op, op.op); + return last; + } + } + + // *** Client OT entrypoints. + + // Submit an operation to the document. + // + // @param operation handled by the OT type + // @param options {source: ...} + // @param [callback] called after operation submitted + // + // @fires before op, op, after op + submitOp(component, options, callback) { + if (typeof options === 'function') { + callback = options; + options = null; + } + var op = {op: component}; + var source = options && options.source; + this._submit(op, source, callback); + } + + // Create the document, which in ShareJS semantics means to set its type. Every + // object implicitly exists in the database but has no data and no type. Create + // sets the type of the object and can optionally set some initial data on the + // object, depending on the type. + // + // @param data initial + // @param type OT type + // @param options {source: ...} + // @param callback called when operation submitted + create(data, type, options, callback) { + if (typeof type === 'function') { + callback = type; + options = null; + type = null; + } else if (typeof options === 'function') { + callback = options; + options = null; + } + if (!type) { + type = types.defaultType.uri; + } + if (this.type) { + var err = new ShareDBError(ERROR_CODE.ERR_DOC_ALREADY_CREATED, 'Document already exists'); + if (callback) return callback(err); + return this.emit('error', err); + } + var op = {create: {type: type, data: data}}; + var source = options && options.source; + this._submit(op, source, callback); + } + + // Delete the document. This creates and submits a delete operation to the + // server. Deleting resets the object's type to null and deletes its data. The + // document still exists, and still has the version it used to have before you + // deleted it (well, old version +1). + // + // @param options {source: ...} + // @param callback called when operation submitted + del(options, callback) { + if (typeof options === 'function') { + callback = options; + options = null; + } + if (!this.type) { + var err = new ShareDBError(ERROR_CODE.ERR_DOC_DOES_NOT_EXIST, 'Document does not exist'); + if (callback) return callback(err); + return this.emit('error', err); + } + var op = {del: true}; + var source = options && options.source; + this._submit(op, source, callback); + } + + // Stops the document from sending any operations to the server. + pause() { + this.paused = true; + } + + // Continue sending operations to the server + resume() { + this.paused = false; + this.flush(); + } + + // Create a snapshot that can be serialized, deserialized, and passed into `Doc.ingestSnapshot`. + toSnapshot() { + return { + v: this.version, + data: clone(this.data), + type: this.type.uri + }; + } + + // *** Receiving operations + + // This is called when the server acknowledges an operation from the client. + _opAcknowledged(message) { + if (this.inflightOp.create) { + this.version = message.v; + } else if (message.v !== this.version) { + // We should already be at the same version, because the server should + // have sent all the ops that have happened before acknowledging our op + logger.warn('Invalid version from server. Expected: ' + this.version + ' Received: ' + message.v, message); + + // Fetching should get us back to a working document state + return this.fetch(); + } + + if (message[ACTIONS.fixup]) { + for (var i = 0; i < message[ACTIONS.fixup].length; i++) { + var fixupOp = message[ACTIONS.fixup][i]; + + for (var j = 0; j < this.pendingOps.length; j++) { + var transformErr = transformX(this.pendingOps[i], fixupOp); + if (transformErr) return this._hardRollback(transformErr); + } + + try { + this._otApply(fixupOp, false); + } catch (error) { + return this._hardRollback(error); + } + } + } + + // The op was committed successfully. Increment the version number + this.version++; + + this._clearInflightOp(); + } + + _rollback(err) { + // The server has rejected submission of the current operation. Invert by + // just the inflight op if possible. If not possible to invert, cancel all + // pending ops and fetch the latest from the server to get us back into a + // working state, then call back + var op = this.inflightOp; + + if (!('op' in op && op.type.invert)) { + return this._hardRollback(err); + } + + try { + op.op = op.type.invert(op.op); + } catch (error) { + // If the op doesn't support `.invert()`, we just reload the doc + // instead of trying to locally revert it. + return this._hardRollback(err); + } + + // Transform the undo operation by any pending ops. + for (var i = 0; i < this.pendingOps.length; i++) { + var transformErr = transformX(this.pendingOps[i], op); + if (transformErr) return this._hardRollback(transformErr); + } + + // ... and apply it locally, reverting the changes. + // + // This operation is applied to look like it comes from a remote source. + // I'm still not 100% sure about this functionality, because its really a + // local op. Basically, the problem is that if the client's op is rejected + // by the server, the editor window should update to reflect the undo. + try { + this._otApply(op, false); + } catch (error) { + return this._hardRollback(error); + } + + // The server has rejected submission of the current operation. If we get + // an "Op submit rejected" error, this was done intentionally + // and we should roll back but not return an error to the user. + if (err.code === ERROR_CODE.ERR_OP_SUBMIT_REJECTED) { + return this._clearInflightOp(null); + } + + this._clearInflightOp(err); + } + + _hardRollback(err) { + this._isInHardRollback = true; + // Store pending ops so that we can notify their callbacks of the error. + // We combine the inflight op and the pending ops, because it's possible + // to hit a condition where we have no inflight op, but we do have pending + // ops. This can happen when an invalid op is submitted, which causes us + // to hard rollback before the pending op was flushed. + var pendingOps = this.pendingOps; + var inflightOp = this.inflightOp; + + // Cancel all pending ops and reset if we can't invert + this._setType(null); + this.version = null; + this.inflightOp = null; + this.pendingOps = []; + + // Fetch the latest version from the server to get us back into a working state + var doc = this; + this._fetch({force: true}, function(fetchError) { + doc._isInHardRollback = false; + + // We want to check that no errors are swallowed, so we check that: + // - there are callbacks to call, and + // - that every single pending op called a callback + // If there are no ops queued, or one of them didn't handle the error, + // then we emit the error. + + if (fetchError) { + // This is critical error as it means that our doc is not in usable state + // anymore, we should throw doc error. + logger.error('Hard rollback doc fetch failed.', fetchError, inflightOp); + + doc.emit('error', new ShareDBError( + ERROR_CODE.ERR_HARD_ROLLBACK_FETCH_FAILED, + 'Hard rollback fetch failed: ' + fetchError.message + )); + } + + if (err.code === ERROR_CODE.ERR_OP_SUBMIT_REJECTED) { + /** + * Handle special case of ERR_OP_SUBMIT_REJECTED + * This ensures that we resolve the main op callback and reject + * all the pending ops. This is hard rollback so all the pending ops will be + * discarded. This will ensure that the user is at least informed about it. + * more info: https://github.com/share/sharedb/pull/626 + */ + if (inflightOp) { + util.callEach(inflightOp.callbacks); + inflightOp = null; + } + + if (!pendingOps.length) return; + err = new ShareDBError( + ERROR_CODE.ERR_PENDING_OP_REMOVED_BY_OP_SUBMIT_REJECTED, + 'Discarding pending op because of hard rollback during ERR_OP_SUBMIT_REJECTED' + ); + } + + if (inflightOp) pendingOps.unshift(inflightOp); + var allOpsHadCallbacks = !!pendingOps.length; + for (var i = 0; i < pendingOps.length; i++) { + allOpsHadCallbacks = util.callEach(pendingOps[i].callbacks, err) && allOpsHadCallbacks; + } + if (err && !allOpsHadCallbacks) doc.emit('error', err); + }); + } + + _clearInflightOp(err) { + var inflightOp = this.inflightOp; + + this.inflightOp = null; + + var called = util.callEach(inflightOp.callbacks, err); + + this.flush(); + this._emitNothingPending(); + + if (err && !called) return this.emit('error', err); + } +} + +emitter.mixin(Doc); + +function combineCallbacks(callbacks) { + callbacks = callbacks.filter(util.truthy); + if (!callbacks.length) return null; + return function(error) { + util.callEach(callbacks, error); + }; +} + + +// Helper function to set op to contain a no-op. +function setNoOp(op) { + delete op.op; + delete op.create; + delete op.del; +} + +// Transform server op data by a client op, and vice versa. Ops are edited in place. +function transformX(client, server) { + // Order of statements in this function matters. Be especially careful if + // refactoring this function + + // A client delete op should dominate if both the server and the client + // delete the document. Thus, any ops following the client delete (such as a + // subsequent create) will be maintained, since the server op is transformed + // to a no-op + if (client.del) return setNoOp(server); + + if (server.del) { + return new ShareDBError(ERROR_CODE.ERR_DOC_WAS_DELETED, 'Document was deleted'); + } + if (server.create) { + return new ShareDBError(ERROR_CODE.ERR_DOC_ALREADY_CREATED, 'Document already created'); + } + + // Ignore no-op coming from server + if (!('op' in server)) return; + + // I believe that this should not occur, but check just in case + if (client.create) { + return new ShareDBError(ERROR_CODE.ERR_DOC_ALREADY_CREATED, 'Document already created'); + } + + // They both edited the document. This is the normal case for this function - + // as in, most of the time we'll end up down here. + // + // You should be wondering why I'm using client.type instead of this.type. + // The reason is, if we get ops at an old version of the document, this.type + // might be undefined or a totally different type. By pinning the type to the + // op data, we make sure the right type has its transform function called. + if (client.type.transformX) { + var result = client.type.transformX(client.op, server.op); + client.op = result[0]; + server.op = result[1]; + } else { + var clientOp = client.type.transform(client.op, server.op, 'left'); + var serverOp = client.type.transform(server.op, client.op, 'right'); + client.op = clientOp; + server.op = serverOp; + } +} + + diff --git a/src/client/index.ts b/src/client/index.ts new file mode 100644 index 000000000..a90ccb22d --- /dev/null +++ b/src/client/index.ts @@ -0,0 +1,6 @@ +export const Connection = require('./connection'); +export const Doc = require('./doc'); +export const Error = require('../error'); +export const Query = require('./query'); +export const types = require('../types'); +export const logger = require('../logger'); diff --git a/src/client/presence/doc-presence-emitter.ts b/src/client/presence/doc-presence-emitter.ts new file mode 100644 index 000000000..086579e6b --- /dev/null +++ b/src/client/presence/doc-presence-emitter.ts @@ -0,0 +1,81 @@ +import util = require('../../util'); +import { EventEmitter } from 'events'; + +var EVENTS = [ + 'create', + 'del', + 'destroy', + 'load', + 'op' +]; + +export = DocPresenceEmitter; + +class DocPresenceEmitter { + _docs; + _forwarders; + _emitters; + + constructor() { + this._docs = Object.create(null); + this._forwarders = Object.create(null); + this._emitters = Object.create(null); + } + + addEventListener(doc, event, listener) { + this._registerDoc(doc); + var emitter = util.dig(this._emitters, doc.collection, doc.id); + emitter.on(event, listener); + } + + removeEventListener(doc, event, listener) { + var emitter = util.dig(this._emitters, doc.collection, doc.id); + if (!emitter) return; + emitter.off(event, listener); + // We'll always have at least one, because of the destroy listener + if (emitter._eventsCount === 1) this._unregisterDoc(doc); + } + + _registerDoc(doc) { + var alreadyRegistered = true; + util.digOrCreate(this._docs, doc.collection, doc.id, function() { + alreadyRegistered = false; + return doc; + }); + + if (alreadyRegistered) return; + + var emitter = util.digOrCreate(this._emitters, doc.collection, doc.id, function() { + var e = new EventEmitter(); + // Set a high limit to avoid unnecessary warnings, but still + // retain some degree of memory leak detection + e.setMaxListeners(1000); + return e; + }); + + var self = this; + EVENTS.forEach(function(event) { + var forwarder = util.digOrCreate(self._forwarders, doc.collection, doc.id, event, function() { + return emitter.emit.bind(emitter, event); + }); + + doc.on(event, forwarder); + }); + + this.addEventListener(doc, 'destroy', this._unregisterDoc.bind(this, doc)); + } + + _unregisterDoc(doc) { + var forwarders = util.dig(this._forwarders, doc.collection, doc.id); + for (var event in forwarders) { + doc.off(event, forwarders[event]); + } + + var emitter = util.dig(this._emitters, doc.collection, doc.id); + emitter.removeAllListeners(); + + util.digAndRemove(this._forwarders, doc.collection, doc.id); + util.digAndRemove(this._emitters, doc.collection, doc.id); + util.digAndRemove(this._docs, doc.collection, doc.id); + } +} diff --git a/src/client/presence/doc-presence.ts b/src/client/presence/doc-presence.ts new file mode 100644 index 000000000..349417968 --- /dev/null +++ b/src/client/presence/doc-presence.ts @@ -0,0 +1,30 @@ +import Presence = require('./presence'); +import LocalDocPresence = require('./local-doc-presence'); +import RemoteDocPresence = require('./remote-doc-presence'); + +class DocPresence extends Presence { + collection; + id; + + constructor(connection, collection, id) { + var channel = DocPresence.channel(collection, id); + super(connection, channel); + + this.collection = collection; + this.id = id; + } + + _createLocalPresence(id) { + return new LocalDocPresence(this, id); + } + + _createRemotePresence(id) { + return new RemoteDocPresence(this, id); + } + + static channel(collection, id) { + return collection + '.' + id; + } +} + +export = DocPresence; diff --git a/src/client/presence/local-doc-presence.ts b/src/client/presence/local-doc-presence.ts new file mode 100644 index 000000000..a498540fc --- /dev/null +++ b/src/client/presence/local-doc-presence.ts @@ -0,0 +1,164 @@ +import LocalPresence = require('./local-presence'); +import ShareDBError = require('../../error'); +import util = require('../../util'); +var ERROR_CODE = ShareDBError.CODES; + +export = LocalDocPresence; + +class LocalDocPresence extends LocalPresence { + collection; + id; + _doc; + _emitter; + _isSending; + _docDataVersionByPresenceVersion; + _opHandler; + _createOrDelHandler; + _loadHandler; + _destroyHandler; + + constructor(presence, presenceId) { + super(presence, presenceId); + + this.collection = this.presence.collection; + this.id = this.presence.id; + + this._doc = this.connection.get(this.collection, this.id); + this._emitter = this.connection._docPresenceEmitter; + this._isSending = false; + this._docDataVersionByPresenceVersion = Object.create(null); + + this._opHandler = this._transformAgainstOp.bind(this); + this._createOrDelHandler = this._handleCreateOrDel.bind(this); + this._loadHandler = this._handleLoad.bind(this); + this._destroyHandler = this.destroy.bind(this); + this._registerWithDoc(); + } + + submit(value, callback) { + if (!this._doc.type) { + // If the Doc hasn't been created, we already assume all presence to + // be null. Let's early return, instead of error since this is a harmless + // no-op + if (value === null) return this._callbackOrEmit(null, callback); + + var error = null; + if (this._doc._isInHardRollback) { + error = { + code: ERROR_CODE.ERR_DOC_IN_HARD_ROLLBACK, + message: 'Cannot submit presence. Document is processing hard rollback' + }; + } else { + error = { + code: ERROR_CODE.ERR_DOC_DOES_NOT_EXIST, + message: 'Cannot submit presence. Document has not been created' + }; + } + + return this._callbackOrEmit(error, callback); + }; + + // Record the current data state version to check if we need to transform + // the presence later + this._docDataVersionByPresenceVersion[this.presenceVersion] = this._doc._dataStateVersion; + LocalPresence.prototype.submit.call(this, value, callback); + } + + destroy(callback) { + this._emitter.removeEventListener(this._doc, 'op', this._opHandler); + this._emitter.removeEventListener(this._doc, 'create', this._createOrDelHandler); + this._emitter.removeEventListener(this._doc, 'del', this._createOrDelHandler); + this._emitter.removeEventListener(this._doc, 'load', this._loadHandler); + this._emitter.removeEventListener(this._doc, 'destroy', this._destroyHandler); + + LocalPresence.prototype.destroy.call(this, callback); + } + + _sendPending() { + if (this._isSending) return; + this._isSending = true; + var presence = this; + this._doc.whenNothingPending(function() { + presence._isSending = false; + if (!presence.connection.canSend) return; + + presence._pendingMessages.forEach(function(message) { + message.t = presence._doc.type.uri; + message.v = presence._doc.version; + presence.connection.send(message); + }); + + presence._pendingMessages = []; + presence._docDataVersionByPresenceVersion = Object.create(null); + }); + } + + _registerWithDoc() { + this._emitter.addEventListener(this._doc, 'op', this._opHandler); + this._emitter.addEventListener(this._doc, 'create', this._createOrDelHandler); + this._emitter.addEventListener(this._doc, 'del', this._createOrDelHandler); + this._emitter.addEventListener(this._doc, 'load', this._loadHandler); + this._emitter.addEventListener(this._doc, 'destroy', this._destroyHandler); + } + + _transformAgainstOp(op, source) { + var presence = this; + var docDataVersion = this._doc._dataStateVersion; + + this._pendingMessages.forEach(function(message) { + // Check if the presence needs transforming against the op - this is to check against + // edge cases where presence is submitted from an 'op' event + var messageDocDataVersion = presence._docDataVersionByPresenceVersion[message.pv]; + if (messageDocDataVersion >= docDataVersion) return; + try { + message.p = presence._transformPresence(message.p, op, source); + // Ensure the presence's data version is kept consistent to deal with "deep" op + // submissions + presence._docDataVersionByPresenceVersion[message.pv] = docDataVersion; + } catch (error) { + var callback = presence._getCallback(message.pv); + presence._callbackOrEmit(error, callback); + } + }); + + try { + this.value = this._transformPresence(this.value, op, source); + } catch (error) { + this.emit('error', error); + } + } + + _handleCreateOrDel() { + this._pendingMessages.forEach(function(message) { + message.p = null; + }); + + this.value = null; + } + + _handleLoad() { + this.value = null; + this._pendingMessages = []; + this._docDataVersionByPresenceVersion = Object.create(null); + } + + _message() { + var message = LocalPresence.prototype._message.call(this); + message.c = this.collection, + message.d = this.id, + message.v = null; + message.t = null; + return message; + } + + _transformPresence(value, op, source) { + var type = this._doc.type; + if (!util.supportsPresence(type)) { + throw new ShareDBError( + ERROR_CODE.ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE, + 'Type does not support presence: ' + type.name + ); + } + return type.transformPresence(value, op, source); + } +} diff --git a/src/client/presence/local-presence.ts b/src/client/presence/local-presence.ts new file mode 100644 index 000000000..d4667ecb3 --- /dev/null +++ b/src/client/presence/local-presence.ts @@ -0,0 +1,92 @@ +import emitter = require('../../emitter'); +import { ACTIONS } from '../../message-actions'; +import util = require('../../util'); + +export = LocalPresence; + +class LocalPresence { + presence; + presenceId; + connection; + presenceVersion; + value; + _pendingMessages; + _callbacksByPresenceVersion; + + constructor(presence, presenceId) { + emitter.EventEmitter.call(this); + + if (!presenceId || typeof presenceId !== 'string') { + throw new Error('LocalPresence presenceId must be a string'); + } + + this.presence = presence; + this.presenceId = presenceId; + this.connection = presence.connection; + this.presenceVersion = 0; + + this.value = null; + + this._pendingMessages = []; + this._callbacksByPresenceVersion = Object.create(null); + } + + submit(value, callback) { + this.value = value; + this.send(callback); + } + + send(callback) { + var message = this._message(); + this._pendingMessages.push(message); + this._callbacksByPresenceVersion[message.pv] = callback; + this._sendPending(); + } + + destroy(callback) { + var presence = this; + this.submit(null, function(error) { + if (error) return presence._callbackOrEmit(error, callback); + delete presence.presence.localPresences[presence.presenceId]; + if (callback) callback(); + }); + } + + _sendPending() { + if (!this.connection.canSend) return; + var presence = this; + this._pendingMessages.forEach(function(message) { + presence.connection.send(message); + }); + + this._pendingMessages = []; + } + + _ack(error, presenceVersion) { + var callback = this._getCallback(presenceVersion); + this._callbackOrEmit(error, callback); + } + + _message() { + return { + a: ACTIONS.presence, + ch: this.presence.channel, + id: this.presenceId, + p: this.value, + pv: this.presenceVersion++ + }; + } + + _getCallback(presenceVersion) { + var callback = this._callbacksByPresenceVersion[presenceVersion]; + delete this._callbacksByPresenceVersion[presenceVersion]; + return callback; + } + + _callbackOrEmit(error, callback) { + if (callback) return util.nextTick(callback, error); + if (error) this.emit('error', error); + } +} + +emitter.mixin(LocalPresence); diff --git a/src/client/presence/presence.ts b/src/client/presence/presence.ts new file mode 100644 index 000000000..15101db0a --- /dev/null +++ b/src/client/presence/presence.ts @@ -0,0 +1,228 @@ +import emitter = require('../../emitter'); +import LocalPresence = require('./local-presence'); +import RemotePresence = require('./remote-presence'); +import util = require('../../util'); +import async = require('async'); +import hat = require('hat'); +import { ACTIONS } from '../../message-actions'; + +export = Presence; + +class Presence { + connection; + channel; + wantSubscribe; + subscribed; + remotePresences; + localPresences; + _remotePresenceInstances; + _subscriptionCallbacksBySeq; + _wantsDestroy; + + constructor(connection, channel) { + emitter.EventEmitter.call(this); + + if (!channel || typeof channel !== 'string') { + throw new Error('Presence channel must be provided'); + } + + this.connection = connection; + this.channel = channel; + + this.wantSubscribe = false; + this.subscribed = false; + this.remotePresences = Object.create(null); + this.localPresences = Object.create(null); + + this._remotePresenceInstances = Object.create(null); + this._subscriptionCallbacksBySeq = Object.create(null); + this._wantsDestroy = false; + } + + subscribe(callback) { + this._sendSubscriptionAction(true, callback); + } + + unsubscribe(callback) { + this._sendSubscriptionAction(false, callback); + } + + create(id) { + if (this._wantsDestroy) { + throw new Error('Presence is being destroyed'); + } + id = id || hat(); + var localPresence = this._createLocalPresence(id); + this.localPresences[id] = localPresence; + return localPresence; + } + + destroy(callback) { + this._wantsDestroy = true; + var presence = this; + // Store these at the time of destruction: any LocalPresence on this + // instance at this time will be destroyed, but if the destroy is + // cancelled, any future LocalPresence objects will be kept. + // See: https://github.com/share/sharedb/pull/579 + var localIds = Object.keys(presence.localPresences); + this.unsubscribe(function(error) { + if (error) return presence._callbackOrEmit(error, callback); + var remoteIds = Object.keys(presence._remotePresenceInstances); + async.parallel( + [ + function(next) { + async.each(localIds, function(presenceId, next) { + var localPresence = presence.localPresences[presenceId]; + if (!localPresence) return next(); + localPresence.destroy(next); + }, next); + }, + function(next) { + // We don't bother stashing the RemotePresence instances because + // they're not really bound to our local state: if we want to + // destroy, we destroy them all, but if we cancel the destroy, + // we'll want to keep them all + if (!presence._wantsDestroy) return next(); + async.each(remoteIds, function(presenceId, next) { + presence._remotePresenceInstances[presenceId].destroy(next); + }, next); + } + ], + function(error) { + if (presence._wantsDestroy) delete presence.connection._presences[presence.channel]; + presence._callbackOrEmit(error, callback); + } + ); + }); + } + + _sendSubscriptionAction(wantSubscribe, callback) { + wantSubscribe = !!wantSubscribe; + if (wantSubscribe === this.wantSubscribe) { + if (!callback) return; + if (wantSubscribe === this.subscribed) return util.nextTick(callback); + if (Object.keys(this._subscriptionCallbacksBySeq).length) { + return this._combineSubscribeCallbackWithLastAdded(callback); + } + } + this.wantSubscribe = wantSubscribe; + var action = this.wantSubscribe ? ACTIONS.presenceSubscribe : ACTIONS.presenceUnsubscribe; + var seq = this.connection._presenceSeq++; + this._subscriptionCallbacksBySeq[seq] = callback; + if (this.connection.canSend) { + this.connection._sendPresenceAction(action, seq, this); + } + } + + _requestRemotePresence() { + this.connection._requestRemotePresence(this.channel); + } + + _handleSubscribe(error, seq) { + if (this.wantSubscribe) this.subscribed = true; + var callback = this._subscriptionCallback(seq); + this._callbackOrEmit(error, callback); + } + + _handleUnsubscribe(error, seq) { + this.subscribed = false; + var callback = this._subscriptionCallback(seq); + this._callbackOrEmit(error, callback); + } + + _receiveUpdate(error, message) { + var localPresence = util.dig(this.localPresences, message.id); + if (localPresence) return localPresence._ack(error, message.pv); + + if (error) return this.emit('error', error); + var presence = this; + var remotePresence = util.digOrCreate(this._remotePresenceInstances, message.id, function() { + return presence._createRemotePresence(message.id); + }); + + remotePresence.receiveUpdate(message); + } + + _updateRemotePresence(remotePresence) { + this.remotePresences[remotePresence.presenceId] = remotePresence.value; + if (remotePresence.value === null) this._removeRemotePresence(remotePresence.presenceId); + this.emit('receive', remotePresence.presenceId, remotePresence.value); + } + + _broadcastAllLocalPresence(error) { + if (error) return this.emit('error', error); + for (var id in this.localPresences) { + var localPresence = this.localPresences[id]; + if (localPresence.value !== null) localPresence.send(); + } + } + + _removeRemotePresence(id) { + this._remotePresenceInstances[id].destroy(); + delete this._remotePresenceInstances[id]; + delete this.remotePresences[id]; + } + + _onConnectionStateChanged() { + if (!this.connection.canSend) { + this.subscribed = false; + return; + } + this._resubscribe(); + for (var id in this.localPresences) { + this.localPresences[id]._sendPending(); + } + } + + _resubscribe() { + var callbacks = []; + for (var seq in this._subscriptionCallbacksBySeq) { + var callback = this._subscriptionCallback(seq); + callbacks.push(callback); + } + + if (!this.wantSubscribe) return this._callEachOrEmit(callbacks); + + var presence = this; + this.subscribe(function(error) { + presence._callEachOrEmit(callbacks, error); + }); + } + + _subscriptionCallback(seq) { + var callback = this._subscriptionCallbacksBySeq[seq]; + delete this._subscriptionCallbacksBySeq[seq]; + return callback; + } + + _callbackOrEmit(error, callback) { + if (callback) return util.nextTick(callback, error); + if (error) this.emit('error', error); + } + + _createLocalPresence(id) { + return new LocalPresence(this, id); + } + + _createRemotePresence(id) { + return new RemotePresence(this, id); + } + + _callEachOrEmit(callbacks, error) { + var called = util.callEach(callbacks, error); + if (!called && error) this.emit('error', error); + } + + _combineSubscribeCallbackWithLastAdded(callback) { + var seqs = Object.keys(this._subscriptionCallbacksBySeq); + var lastSeq = seqs[seqs.length - 1]; + var originalCallback = this._subscriptionCallbacksBySeq[lastSeq]; + if (!originalCallback) return this._subscriptionCallbacksBySeq[lastSeq] = callback; + this._subscriptionCallbacksBySeq[lastSeq] = function(error) { + originalCallback(error); + callback(error); + }; + } +} + +emitter.mixin(Presence); diff --git a/src/client/presence/remote-doc-presence.ts b/src/client/presence/remote-doc-presence.ts new file mode 100644 index 000000000..1b8bb6f96 --- /dev/null +++ b/src/client/presence/remote-doc-presence.ts @@ -0,0 +1,164 @@ +import RemotePresence = require('./remote-presence'); +import ot = require('../../ot'); + +export = RemoteDocPresence; + +class RemoteDocPresence extends RemotePresence { + collection; + id; + src; + presenceVersion; + _doc; + _emitter; + _pending; + _opCache; + _pendingSetPending; + _opHandler; + _createDelHandler; + _loadHandler; + + constructor(presence, presenceId) { + super(presence, presenceId); + + this.collection = this.presence.collection; + this.id = this.presence.id; + this.src = null; + this.presenceVersion = null; + + this._doc = this.connection.get(this.collection, this.id); + this._emitter = this.connection._docPresenceEmitter; + this._pending = null; + this._opCache = null; + this._pendingSetPending = false; + + this._opHandler = this._handleOp.bind(this); + this._createDelHandler = this._handleCreateDel.bind(this); + this._loadHandler = this._handleLoad.bind(this); + this._registerWithDoc(); + } + + receiveUpdate(message) { + if (this._pending && message.pv < this._pending.pv) return; + this.src = message.src; + this._pending = message; + this._setPendingPresence(); + } + + destroy(callback) { + this._emitter.removeEventListener(this._doc, 'op', this._opHandler); + this._emitter.removeEventListener(this._doc, 'create', this._createDelHandler); + this._emitter.removeEventListener(this._doc, 'del', this._createDelHandler); + this._emitter.removeEventListener(this._doc, 'load', this._loadHandler); + + RemotePresence.prototype.destroy.call(this, callback); + } + + _registerWithDoc() { + this._emitter.addEventListener(this._doc, 'op', this._opHandler); + this._emitter.addEventListener(this._doc, 'create', this._createDelHandler); + this._emitter.addEventListener(this._doc, 'del', this._createDelHandler); + this._emitter.addEventListener(this._doc, 'load', this._loadHandler); + } + + _setPendingPresence() { + if (this._pendingSetPending) return; + this._pendingSetPending = true; + var presence = this; + this._doc.whenNothingPending(function() { + presence._pendingSetPending = false; + if (!presence._pending) return; + if (presence._pending.pv < presence.presenceVersion) return presence._pending = null; + + if (presence._pending.v > presence._doc.version) { + return presence._doc.fetch(); + } + + if (!presence._catchUpStalePresence()) return; + + presence.value = presence._pending.p; + presence.presenceVersion = presence._pending.pv; + presence._pending = null; + presence.presence._updateRemotePresence(presence); + }); + } + + _handleOp(op, source, connectionId) { + var isOwnOp = connectionId === this.src; + this._transformAgainstOp(op, isOwnOp); + this._cacheOp(op, isOwnOp); + this._setPendingPresence(); + } + + _transformAgainstOp(op, isOwnOp) { + if (!this.value) return; + + try { + this.value = this._doc.type.transformPresence(this.value, op, isOwnOp); + } catch (error) { + return this.presence.emit('error', error); + } + this.presence._updateRemotePresence(this); + } + + _catchUpStalePresence() { + if (this._pending.v >= this._doc.version) return true; + + if (!this._opCache) { + this._startCachingOps(); + this._doc.fetch(); + this.presence._requestRemotePresence(); + return false; + } + + while (this._opCache[this._pending.v]) { + var item = this._opCache[this._pending.v]; + var op = item.op; + var isOwnOp = item.isOwnOp; + // We use a null op to signify a create or a delete operation. In both + // cases we just want to reset the presence (which doesn't make sense + // in a new document), so just set the presence to null. + if (op === null) { + this._pending.p = null; + this._pending.v++; + } else { + ot.transformPresence(this._pending, op, isOwnOp); + } + } + + var hasCaughtUp = this._pending.v >= this._doc.version; + if (hasCaughtUp) { + this._stopCachingOps(); + } + + return hasCaughtUp; + } + + _startCachingOps() { + this._opCache = []; + } + + _stopCachingOps() { + this._opCache = null; + } + + _cacheOp(op, isOwnOp) { + if (this._opCache) { + op = op ? {op: op} : null; + // Subtract 1 from the current doc version, because an op with v3 + // should be read as the op that takes a doc from v3 -> v4 + this._opCache[this._doc.version - 1] = {op: op, isOwnOp: isOwnOp}; + } + } +} + +RemotePresence.prototype._handleCreateDel = function() { + this._cacheOp(null); + this._setPendingPresence(); +}; + +RemotePresence.prototype._handleLoad = function() { + this.value = null; + this._pending = null; + this._opCache = null; + this.presence._updateRemotePresence(this); +}; diff --git a/src/client/presence/remote-presence.ts b/src/client/presence/remote-presence.ts new file mode 100644 index 000000000..eefdff45d --- /dev/null +++ b/src/client/presence/remote-presence.ts @@ -0,0 +1,33 @@ +import util = require('../../util'); + +export = RemotePresence; + +class RemotePresence { + presence; + presenceId; + connection; + value; + presenceVersion; + + constructor(presence, presenceId) { + this.presence = presence; + this.presenceId = presenceId; + this.connection = this.presence.connection; + + this.value = null; + this.presenceVersion = 0; + } + + receiveUpdate(message) { + if (message.pv < this.presenceVersion) return; + this.value = message.p; + this.presenceVersion = message.pv; + this.presence._updateRemotePresence(this); + } + + destroy(callback) { + delete this.presence._remotePresenceInstances[this.presenceId]; + delete this.presence.remotePresences[this.presenceId]; + if (callback) util.nextTick(callback); + } +} diff --git a/src/client/query.ts b/src/client/query.ts new file mode 100644 index 000000000..7b5ea6ec8 --- /dev/null +++ b/src/client/query.ts @@ -0,0 +1,220 @@ +import emitter = require('../emitter'); +import { ACTIONS } from '../message-actions'; +import util = require('../util'); + +// Queries are live requests to the database for particular sets of fields. +// +// The server actively tells the client when there's new data that matches +// a set of conditions. +// Queries are live requests to the database for particular sets of fields. +// +// The server actively tells the client when there's new data that matches +// a set of conditions. +export = Query; + +class Query { + action; + connection; + id; + collection; + query; + results; + extra; + options; + callback; + ready; + sent; + + constructor(action, connection, id, collection, query, options, callback) { + emitter.EventEmitter.call(this); + + // 'qf' or 'qs' + this.action = action; + + this.connection = connection; + this.id = id; + this.collection = collection; + + // The query itself. For mongo, this should look something like {"data.x":5} + this.query = query; + + // A list of resulting documents. These are actual documents, complete with + // data and all the rest. It is possible to pass in an initial results set, + // so that a query can be serialized and then re-established + this.results = null; + if (options && options.results) { + this.results = options.results; + delete options.results; + } + this.extra = undefined; + + // Options to pass through with the query + this.options = options; + + this.callback = callback; + this.ready = false; + this.sent = false; + } + + hasPending() { + return !this.ready; + } + + // Helper for subscribe & fetch, since they share the same message format. + // + // This function actually issues the query. + send() { + if (!this.connection.canSend) return; + + var message = { + a: this.action, + id: this.id, + c: this.collection, + q: this.query + }; + if (this.options) { + message.o = this.options; + } + if (this.results) { + // Collect the version of all the documents in the current result set so we + // don't need to be sent their snapshots again. + var results = []; + for (var i = 0; i < this.results.length; i++) { + var doc = this.results[i]; + results.push([doc.id, doc.version]); + } + message.r = results; + } + + this.connection.send(message); + this.sent = true; + } + + // Destroy the query object. Any subsequent messages for the query will be + // ignored by the connection. + destroy(callback) { + if (this.connection.canSend && this.action === ACTIONS.querySubscribe) { + this.connection.send({a: ACTIONS.queryUnsubscribe, id: this.id}); + } + this.connection._destroyQuery(this); + // There is a callback for consistency, but we don't actually wait for the + // server's unsubscribe message currently + if (callback) util.nextTick(callback); + } + + _onConnectionStateChanged() { + if (this.connection.canSend && !this.sent) { + this.send(); + } else { + this.sent = false; + } + } + + _handleFetch(err, data, extra) { + // Once a fetch query gets its data, it is destroyed. + this.connection._destroyQuery(this); + this._handleResponse(err, data, extra); + } + + _handleSubscribe(err, data, extra) { + this._handleResponse(err, data, extra); + } + + _handleResponse(err, data, extra) { + var callback = this.callback; + this.callback = null; + if (err) return this._finishResponse(err, callback); + if (!data) return this._finishResponse(null, callback); + + var query = this; + var wait = 1; + var finish = function(err) { + if (err) return query._finishResponse(err, callback); + if (--wait) return; + query._finishResponse(null, callback); + }; + + if (Array.isArray(data)) { + wait += data.length; + this.results = this._ingestSnapshots(data, finish); + this.extra = extra; + } else { + for (var id in data) { + wait++; + var snapshot = data[id]; + var doc = this.connection.get(snapshot.c || this.collection, id); + doc.ingestSnapshot(snapshot, finish); + } + } + + finish(); + } + + _ingestSnapshots(snapshots, finish) { + var results = []; + for (var i = 0; i < snapshots.length; i++) { + var snapshot = snapshots[i]; + var doc = this.connection.get(snapshot.c || this.collection, snapshot.d); + doc.ingestSnapshot(snapshot, finish); + results.push(doc); + } + return results; + } + + _finishResponse(err, callback) { + this.emit('ready'); + this.ready = true; + if (err) { + this.connection._destroyQuery(this); + if (callback) return callback(err); + return this.emit('error', err); + } + if (callback) callback(null, this.results, this.extra); + } + + _handleError(err) { + this.emit('error', err); + } + + _handleDiff(diff) { + // We need to go through the list twice. First, we'll ingest all the new + // documents. After that we'll emit events and actually update our list. + // This avoids race conditions around setting documents to be subscribed & + // unsubscribing documents in event callbacks. + for (var i = 0; i < diff.length; i++) { + var d = diff[i]; + if (d.type === 'insert') d.values = this._ingestSnapshots(d.values); + } + + for (var i = 0; i < diff.length; i++) { + var d = diff[i]; + switch (d.type) { + case 'insert': + var newDocs = d.values; + Array.prototype.splice.apply(this.results, [d.index, 0].concat(newDocs)); + this.emit('insert', newDocs, d.index); + break; + case 'remove': + var howMany = d.howMany || 1; + var removed = this.results.splice(d.index, howMany); + this.emit('remove', removed, d.index); + break; + case 'move': + var howMany = d.howMany || 1; + var docs = this.results.splice(d.from, howMany); + Array.prototype.splice.apply(this.results, [d.to, 0].concat(docs)); + this.emit('move', docs, d.from, d.to); + break; + } + } + + this.emit('changed', this.results); + } + + _handleExtra(extra) { + this.extra = extra; + this.emit('extra', extra); + } +} + +emitter.mixin(Query); diff --git a/src/client/snapshot-request/snapshot-request.ts b/src/client/snapshot-request/snapshot-request.ts new file mode 100644 index 000000000..b6571e1d1 --- /dev/null +++ b/src/client/snapshot-request/snapshot-request.ts @@ -0,0 +1,64 @@ +import Snapshot = require('../../snapshot'); +import emitter = require('../../emitter'); + +export = SnapshotRequest; + +class SnapshotRequest { + requestId; + connection; + id; + collection; + callback; + sent; + + constructor(connection, requestId, collection, id, callback) { + emitter.EventEmitter.call(this); + + if (typeof callback !== 'function') { + throw new Error('Callback is required for SnapshotRequest'); + } + + this.requestId = requestId; + this.connection = connection; + this.id = id; + this.collection = collection; + this.callback = callback; + + this.sent = false; + } + + send() { + if (!this.connection.canSend) { + return; + } + + this.connection.send(this._message()); + this.sent = true; + } + + _onConnectionStateChanged() { + if (this.connection.canSend) { + if (!this.sent) this.send(); + } else { + // If the connection can't send, then we've had a disconnection, and even if we've already sent + // the request previously, we need to re-send it over this reconnected client, so reset the + // sent flag to false. + this.sent = false; + } + } + + _handleResponse(error, message) { + this.emit('ready'); + + if (error) { + return this.callback(error); + } + + var metadata = message.meta ? message.meta : null; + var snapshot = new Snapshot(this.id, message.v, message.type, message.data, metadata); + + this.callback(null, snapshot); + } +} + +emitter.mixin(SnapshotRequest); diff --git a/src/client/snapshot-request/snapshot-timestamp-request.ts b/src/client/snapshot-request/snapshot-timestamp-request.ts new file mode 100644 index 000000000..8966ab903 --- /dev/null +++ b/src/client/snapshot-request/snapshot-timestamp-request.ts @@ -0,0 +1,29 @@ +import SnapshotRequest = require('./snapshot-request'); +import util = require('../../util'); +import { ACTIONS } from '../../message-actions'; + +export = SnapshotTimestampRequest; + +class SnapshotTimestampRequest extends SnapshotRequest { + timestamp; + + constructor(connection, requestId, collection, id, timestamp, callback) { + super(connection, requestId, collection, id, callback); + + if (!util.isValidTimestamp(timestamp)) { + throw new Error('Snapshot timestamp must be a positive integer or null'); + } + + this.timestamp = timestamp; + } + + _message() { + return { + a: ACTIONS.snapshotFetchByTimestamp, + id: this.requestId, + c: this.collection, + d: this.id, + ts: this.timestamp + }; + } +} diff --git a/src/client/snapshot-request/snapshot-version-request.ts b/src/client/snapshot-request/snapshot-version-request.ts new file mode 100644 index 000000000..8ba8309f5 --- /dev/null +++ b/src/client/snapshot-request/snapshot-version-request.ts @@ -0,0 +1,29 @@ +import SnapshotRequest = require('./snapshot-request'); +import util = require('../../util'); +import { ACTIONS } from '../../message-actions'; + +export = SnapshotVersionRequest; + +class SnapshotVersionRequest extends SnapshotRequest { + version; + + constructor(connection, requestId, collection, id, version, callback) { + super(connection, requestId, collection, id, callback); + + if (!util.isValidVersion(version)) { + throw new Error('Snapshot version must be a positive integer or null'); + } + + this.version = version; + } + + _message() { + return { + a: ACTIONS.snapshotFetch, + id: this.requestId, + c: this.collection, + d: this.id, + v: this.version + }; + } +} diff --git a/src/db/index.ts b/src/db/index.ts new file mode 100644 index 000000000..e71f77ec9 --- /dev/null +++ b/src/db/index.ts @@ -0,0 +1,112 @@ +import async = require('async'); +import ShareDBError = require('../error'); + +var ERROR_CODE = ShareDBError.CODES; + +class DB { + pollDebounce; + + constructor(options) { + // pollDebounce is the minimum time in ms between query polls + this.pollDebounce = options && options.pollDebounce; + } + + close(callback) { + if (callback) callback(); + } + + commit(collection, id, op, snapshot, options, callback) { + callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, 'commit DB method unimplemented')); + } + + getSnapshot(collection, id, fields, options, callback) { + callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, 'getSnapshot DB method unimplemented')); + } + + getSnapshotBulk(collection, ids, fields, options, callback) { + var results = Object.create(null); + var db = this; + async.each(ids, function(id, eachCb) { + db.getSnapshot(collection, id, fields, options, function(err, snapshot) { + if (err) return eachCb(err); + results[id] = snapshot; + eachCb(); + }); + }, function(err) { + if (err) return callback(err); + callback(null, results); + }); + } + + getOps(collection, id, from, to, options, callback) { + callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, 'getOps DB method unimplemented')); + } + + getOpsToSnapshot(collection, id, from, snapshot, options, callback) { + var to = snapshot.v; + this.getOps(collection, id, from, to, options, callback); + } + + getOpsBulk(collection, fromMap, toMap, options, callback) { + var results = Object.create(null); + var db = this; + async.forEachOf(fromMap, function(from, id, eachCb) { + var to = toMap && toMap[id]; + db.getOps(collection, id, from, to, options, function(err, ops) { + if (err) return eachCb(err); + results[id] = ops; + eachCb(); + }); + }, function(err) { + if (err) return callback(err); + callback(null, results); + }); + } + + getCommittedOpVersion(collection, id, snapshot, op, options, callback) { + this.getOpsToSnapshot(collection, id, 0, snapshot, options, function(err, ops) { + if (err) return callback(err); + for (var i = ops.length; i--;) { + var item = ops[i]; + if (op.src === item.src && op.seq === item.seq) { + return callback(null, item.v); + } + } + callback(); + }); + } + + query(collection, query, fields, options, callback) { + callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, 'query DB method unimplemented')); + } + + queryPoll(collection, query, options, callback) { + var fields = Object.create(null); + this.query(collection, query, fields, options, function(err, snapshots, extra) { + if (err) return callback(err); + var ids = []; + for (var i = 0; i < snapshots.length; i++) { + ids.push(snapshots[i].id); + } + callback(null, ids, extra); + }); + } + + queryPollDoc(collection, id, query, options, callback) { + callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, 'queryPollDoc DB method unimplemented')); + } + + canPollDoc() { + return false; + } + + skipPoll() { + return false; + } +} + +export = DB; + +// When false, Backend will handle projections instead of DB +DB.prototype.projectsSnapshots = false; +DB.prototype.disableSubscribe = false; diff --git a/src/db/memory.ts b/src/db/memory.ts new file mode 100644 index 000000000..52e58d61c --- /dev/null +++ b/src/db/memory.ts @@ -0,0 +1,193 @@ +import DB = require('./index'); +import Snapshot = require('../snapshot'); +import util = require('../util'); +var clone = util.clone; + +// In-memory ShareDB database +// +// This adapter is not appropriate for production use. It is intended for +// testing and as an API example for people implementing database adaptors. It +// is fully functional, except it stores all documents & operations forever in +// memory. As such, memory usage will grow without bound, it doesn't scale +// across multiple node processes and you'll lose all your data if the server +// restarts. Query APIs are adapter specific. Use with care. + +class MemoryDB extends DB { + docs; + ops; + closed; + + constructor(options) { + super(options); + + // Map from collection name -> doc id -> doc snapshot ({v:, type:, data:}) + this.docs = Object.create(null); + + // Map from collection name -> doc id -> list of operations. Operations + // don't store their version - instead their version is simply the index in + // the list. + this.ops = Object.create(null); + + this.closed = false; + } + + close(callback) { + this.closed = true; + if (callback) callback(); + } + + // Persists an op and snapshot if it is for the next version. Calls back with + // callback(err, succeeded) + commit(collection, id, op, snapshot, options, callback) { + var db = this; + if (typeof callback !== 'function') throw new Error('Callback required'); + util.nextTick(function() { + var version = db._getVersionSync(collection, id); + if (snapshot.v !== version + 1) { + var succeeded = false; + return callback(null, succeeded); + } + var err = db._writeOpSync(collection, id, op); + if (err) return callback(err); + err = db._writeSnapshotSync(collection, id, snapshot); + if (err) return callback(err); + + var succeeded = true; + callback(null, succeeded); + }); + } + + // Get the named document from the database. The callback is called with (err, + // snapshot). A snapshot with a version of zero is returned if the docuemnt + // has never been created in the database. + getSnapshot(collection, id, fields, options, callback) { + var includeMetadata = (fields && fields.$submit) || (options && options.metadata); + var db = this; + if (typeof callback !== 'function') throw new Error('Callback required'); + util.nextTick(function() { + var snapshot = db._getSnapshotSync(collection, id, includeMetadata); + callback(null, snapshot); + }); + } + + // Get operations between [from, to) noninclusively. (Ie, the range should + // contain start but not end). + // + // If end is null, this function should return all operations from start onwards. + // + // The operations that getOps returns don't need to have a version: field. + // The version will be inferred from the parameters if it is missing. + // + // Callback should be called as callback(error, [list of ops]); + getOps(collection, id, from, to, options, callback) { + var includeMetadata = options && options.metadata; + var db = this; + if (typeof callback !== 'function') throw new Error('Callback required'); + util.nextTick(function() { + var opLog = db._getOpLogSync(collection, id); + if (!from) from = 0; + if (to == null) to = opLog.length; + var ops = clone(opLog.slice(from, to).filter(Boolean)); + if (ops.length < to - from) { + return callback(new Error('Missing ops')); + } + if (!includeMetadata) { + for (var i = 0; i < ops.length; i++) { + delete ops[i].m; + } + } + callback(null, ops); + }); + } + + deleteOps(collection, id, from, to, options, callback) { + if (typeof callback !== 'function') throw new Error('Callback required'); + var db = this; + util.nextTick(function() { + var opLog = db._getOpLogSync(collection, id); + if (!from) from = 0; + if (to == null) to = opLog.length; + for (var i = from; i < to; i++) opLog[i] = null; + callback(null); + }); + } + + // The memory database query function returns all documents in a collection + // regardless of query by default + query(collection, query, fields, options, callback) { + var includeMetadata = options && options.metadata; + var db = this; + if (typeof callback !== 'function') throw new Error('Callback required'); + util.nextTick(function() { + var collectionDocs = db.docs[collection]; + var snapshots = []; + for (var id in collectionDocs || {}) { + var snapshot = db._getSnapshotSync(collection, id, includeMetadata); + snapshots.push(snapshot); + } + try { + var result = db._querySync(snapshots, query, options); + callback(null, result.snapshots, result.extra); + } catch (err) { + callback(err); + } + }); + } + + // For testing, it may be useful to implement the desired query + // language by defining this function. Returns an object with + // two properties: + // - snapshots: array of query result snapshots + // - extra: (optional) other types of results, such as counts + _querySync(snapshots) { + return {snapshots: snapshots}; + } + + _writeOpSync(collection, id, op) { + var opLog = this._getOpLogSync(collection, id); + // This will write an op in the log at its version, which should always be + // the next item in the array under normal operation + opLog[op.v] = clone(op); + } + + // Create, update, and delete snapshots. For creates and updates, a snapshot + // object will be passed in with a type property. If there is no type property, + // it should be considered a delete + _writeSnapshotSync(collection, id, snapshot) { + var collectionDocs = this.docs[collection] || (this.docs[collection] = Object.create(null)); + if (!snapshot.type) { + delete collectionDocs[id]; + } else { + collectionDocs[id] = clone(snapshot); + } + } + + _getSnapshotSync(collection, id, includeMetadata) { + var collectionDocs = this.docs[collection]; + // We need to clone the snapshot, because ShareDB assumes each call to + // getSnapshot returns a new object + var doc = collectionDocs && collectionDocs[id]; + var snapshot; + if (doc) { + var data = clone(doc.data); + var meta = (includeMetadata) ? clone(doc.m) : null; + snapshot = new Snapshot(id, doc.v, doc.type, data, meta); + } else { + var version = this._getVersionSync(collection, id); + snapshot = new Snapshot(id, version, null, undefined, null); + } + return snapshot; + } + + _getOpLogSync(collection, id) { + var collectionOps = this.ops[collection] || (this.ops[collection] = Object.create(null)); + return collectionOps[id] || (collectionOps[id] = []); + } + + _getVersionSync(collection, id) { + var collectionOps = this.ops[collection]; + return (collectionOps && collectionOps[id] && collectionOps[id].length) || 0; + } +} + +export = MemoryDB; diff --git a/lib/emitter.js b/src/emitter.ts similarity index 56% rename from lib/emitter.js rename to src/emitter.ts index 2b7827ccd..257cd46d6 100644 --- a/lib/emitter.js +++ b/src/emitter.ts @@ -1,10 +1,9 @@ -var EventEmitter = require('events').EventEmitter; - -exports.EventEmitter = EventEmitter; -exports.mixin = mixin; - -function mixin(Constructor) { - for (var key in EventEmitter.prototype) { - Constructor.prototype[key] = EventEmitter.prototype[key]; - } -} +import { EventEmitter } from 'events'; +export { EventEmitter }; +export { mixin }; + +function mixin(Constructor) { + for (var key in EventEmitter.prototype) { + Constructor.prototype[key] = EventEmitter.prototype[key]; + } +} diff --git a/src/error.ts b/src/error.ts new file mode 100644 index 000000000..1d5b69ed2 --- /dev/null +++ b/src/error.ts @@ -0,0 +1,88 @@ +class ShareDBError { + code; + message; + stack; + + constructor(code, message) { + this.code = code; + this.message = message || ''; + if (Error.captureStackTrace) { + Error.captureStackTrace(this, ShareDBError); + } else { + this.stack = new Error().stack; + } + } + + static CODES = { + ERR_APPLY_OP_VERSION_DOES_NOT_MATCH_SNAPSHOT: 'ERR_APPLY_OP_VERSION_DOES_NOT_MATCH_SNAPSHOT', + ERR_APPLY_SNAPSHOT_NOT_PROVIDED: 'ERR_APPLY_SNAPSHOT_NOT_PROVIDED', + ERR_FIXUP_IS_ONLY_VALID_ON_APPLY: 'ERR_FIXUP_IS_ONLY_VALID_ON_APPLY', + ERR_CANNOT_FIXUP_DELETION: 'ERR_CANNOT_FIXUP_DELETION', + ERR_CLIENT_ID_BADLY_FORMED: 'ERR_CLIENT_ID_BADLY_FORMED', + ERR_CANNOT_PING_OFFLINE: 'ERR_CANNOT_PING_OFFLINE', + ERR_CONNECTION_SEQ_INTEGER_OVERFLOW: 'ERR_CONNECTION_SEQ_INTEGER_OVERFLOW', + ERR_CONNECTION_STATE_TRANSITION_INVALID: 'ERR_CONNECTION_STATE_TRANSITION_INVALID', + ERR_DATABASE_ADAPTER_NOT_FOUND: 'ERR_DATABASE_ADAPTER_NOT_FOUND', + ERR_DATABASE_DOES_NOT_SUPPORT_SUBSCRIBE: 'ERR_DATABASE_DOES_NOT_SUPPORT_SUBSCRIBE', + ERR_DATABASE_METHOD_NOT_IMPLEMENTED: 'ERR_DATABASE_METHOD_NOT_IMPLEMENTED', + ERR_DEFAULT_TYPE_MISMATCH: 'ERR_DEFAULT_TYPE_MISMATCH', + ERR_DOC_MISSING_VERSION: 'ERR_DOC_MISSING_VERSION', + ERR_DOC_ALREADY_CREATED: 'ERR_DOC_ALREADY_CREATED', + ERR_DOC_DOES_NOT_EXIST: 'ERR_DOC_DOES_NOT_EXIST', + ERR_DOC_TYPE_NOT_RECOGNIZED: 'ERR_DOC_TYPE_NOT_RECOGNIZED', + ERR_DOC_WAS_DELETED: 'ERR_DOC_WAS_DELETED', + ERR_DOC_IN_HARD_ROLLBACK: 'ERR_DOC_IN_HARD_ROLLBACK', + ERR_INFLIGHT_OP_MISSING: 'ERR_INFLIGHT_OP_MISSING', + ERR_INGESTED_SNAPSHOT_HAS_NO_VERSION: 'ERR_INGESTED_SNAPSHOT_HAS_NO_VERSION', + ERR_MAX_SUBMIT_RETRIES_EXCEEDED: 'ERR_MAX_SUBMIT_RETRIES_EXCEEDED', + ERR_MESSAGE_BADLY_FORMED: 'ERR_MESSAGE_BADLY_FORMED', + ERR_MILESTONE_ARGUMENT_INVALID: 'ERR_MILESTONE_ARGUMENT_INVALID', + ERR_NO_OP: 'ERR_NO_OP', + ERR_OP_ALREADY_SUBMITTED: 'ERR_OP_ALREADY_SUBMITTED', + ERR_OP_NOT_ALLOWED_IN_PROJECTION: 'ERR_OP_NOT_ALLOWED_IN_PROJECTION', + ERR_OP_SUBMIT_REJECTED: 'ERR_OP_SUBMIT_REJECTED', + ERR_PENDING_OP_REMOVED_BY_OP_SUBMIT_REJECTED: 'ERR_PENDING_OP_REMOVED_BY_OP_SUBMIT_REJECTED', + ERR_HARD_ROLLBACK_FETCH_FAILED: 'ERR_HARD_ROLLBACK_FETCH_FAILED', + ERR_OP_VERSION_MISMATCH_AFTER_TRANSFORM: 'ERR_OP_VERSION_MISMATCH_AFTER_TRANSFORM', + ERR_OP_VERSION_MISMATCH_DURING_TRANSFORM: 'ERR_OP_VERSION_MISMATCH_DURING_TRANSFORM', + ERR_OP_VERSION_NEWER_THAN_CURRENT_SNAPSHOT: 'ERR_OP_VERSION_NEWER_THAN_CURRENT_SNAPSHOT', + ERR_OT_LEGACY_JSON0_OP_CANNOT_BE_NORMALIZED: 'ERR_OT_LEGACY_JSON0_OP_CANNOT_BE_NORMALIZED', + ERR_OT_OP_BADLY_FORMED: 'ERR_OT_OP_BADLY_FORMED', + ERR_OT_OP_NOT_APPLIED: 'ERR_OT_OP_NOT_APPLIED', + ERR_OT_OP_NOT_PROVIDED: 'ERR_OT_OP_NOT_PROVIDED', + ERR_PRESENCE_TRANSFORM_FAILED: 'ERR_PRESENCE_TRANSFORM_FAILED', + ERR_PROTOCOL_VERSION_NOT_SUPPORTED: 'ERR_PROTOCOL_VERSION_NOT_SUPPORTED', + ERR_QUERY_CHANNEL_MISSING: 'ERR_QUERY_CHANNEL_MISSING', + ERR_QUERY_EMITTER_LISTENER_NOT_ASSIGNED: 'ERR_QUERY_EMITTER_LISTENER_NOT_ASSIGNED', + /** + * A special error that a "readSnapshots" middleware implementation can use to indicate that it + * wishes for the ShareDB client to treat it as a silent rejection, not passing the error back to + * user code. + * + * For subscribes, the ShareDB client will still cancel the document subscription. + */ + ERR_SNAPSHOT_READ_SILENT_REJECTION: 'ERR_SNAPSHOT_READ_SILENT_REJECTION', + /** + * A "readSnapshots" middleware rejected the reads of specific snapshots. + * + * This error code is mostly for server use and generally will not be encountered on the client. + * Instead, each specific doc that encountered an error will receive its specific error. + * + * The one exception is for queries, where a "readSnapshots" rejection of specific snapshots will + * cause the client to receive this error for the whole query, since queries don't support + * doc-specific errors. + */ + ERR_SNAPSHOT_READS_REJECTED: 'ERR_SNAPSHOT_READS_REJECTED', + ERR_SUBMIT_TRANSFORM_OPS_NOT_FOUND: 'ERR_SUBMIT_TRANSFORM_OPS_NOT_FOUND', + ERR_TYPE_CANNOT_BE_PROJECTED: 'ERR_TYPE_CANNOT_BE_PROJECTED', + ERR_TYPE_DOES_NOT_SUPPORT_COMPOSE: 'ERR_TYPE_DOES_NOT_SUPPORT_COMPOSE', + ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE: 'ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE', + ERR_UNKNOWN_ERROR: 'ERR_UNKNOWN_ERROR' + }; +} + +ShareDBError.prototype = Object.create(Error.prototype); +ShareDBError.prototype.constructor = ShareDBError; +ShareDBError.prototype.name = 'ShareDBError'; + +export = ShareDBError; diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 000000000..f1b2485e5 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,19 @@ +import Backend = require('./backend'); +export = Object.assign(Backend, { + Agent: require('./agent'), + Backend: Backend, + DB: require('./db'), + Error: require('./error'), + logger: require('./logger'), + MemoryDB: require('./db/memory'), + MemoryMilestoneDB: require('./milestone-db/memory'), + MemoryPubSub: require('./pubsub/memory'), + MESSAGE_ACTIONS: require('./message-actions').ACTIONS, + MilestoneDB: require('./milestone-db'), + ot: require('./ot'), + projections: require('./projections'), + PubSub: require('./pubsub'), + QueryEmitter: require('./query-emitter'), + SubmitRequest: require('./submit-request'), + types: require('./types') +}); diff --git a/src/logger/index.ts b/src/logger/index.ts new file mode 100644 index 000000000..454e7b880 --- /dev/null +++ b/src/logger/index.ts @@ -0,0 +1,3 @@ +import Logger = require('./logger'); +var logger = new Logger(); +export = logger; diff --git a/src/logger/logger.ts b/src/logger/logger.ts new file mode 100644 index 000000000..5dd100c10 --- /dev/null +++ b/src/logger/logger.ts @@ -0,0 +1,29 @@ +var SUPPORTED_METHODS = [ + 'info', + 'warn', + 'error' +]; + +class Logger { + constructor() { + var defaultMethods = Object.create(null); + SUPPORTED_METHODS.forEach(function(method) { + // Deal with Chrome issue: https://bugs.chromium.org/p/chromium/issues/detail?id=179628 + defaultMethods[method] = console[method].bind(console); + }); + this.setMethods(defaultMethods); + } + + setMethods(overrides) { + overrides = overrides || {}; + var logger = this; + + SUPPORTED_METHODS.forEach(function(method) { + if (typeof overrides[method] === 'function') { + logger[method] = overrides[method]; + } + }); + } +} + +export = Logger; diff --git a/lib/message-actions.js b/src/message-actions.ts similarity index 90% rename from lib/message-actions.js rename to src/message-actions.ts index e1a8e9942..a798c60c6 100644 --- a/lib/message-actions.js +++ b/src/message-actions.ts @@ -1,23 +1,23 @@ -exports.ACTIONS = { - initLegacy: 'init', - handshake: 'hs', - queryFetch: 'qf', - querySubscribe: 'qs', - queryUnsubscribe: 'qu', - queryUpdate: 'q', - bulkFetch: 'bf', - bulkSubscribe: 'bs', - bulkUnsubscribe: 'bu', - fetch: 'f', - fixup: 'fixup', - subscribe: 's', - unsubscribe: 'u', - op: 'op', - snapshotFetch: 'nf', - snapshotFetchByTimestamp: 'nt', - pingPong: 'pp', - presence: 'p', - presenceSubscribe: 'ps', - presenceUnsubscribe: 'pu', - presenceRequest: 'pr' -}; +export const ACTIONS = { + initLegacy: 'init', + handshake: 'hs', + queryFetch: 'qf', + querySubscribe: 'qs', + queryUnsubscribe: 'qu', + queryUpdate: 'q', + bulkFetch: 'bf', + bulkSubscribe: 'bs', + bulkUnsubscribe: 'bu', + fetch: 'f', + fixup: 'fixup', + subscribe: 's', + unsubscribe: 'u', + op: 'op', + snapshotFetch: 'nf', + snapshotFetchByTimestamp: 'nt', + pingPong: 'pp', + presence: 'p', + presenceSubscribe: 'ps', + presenceUnsubscribe: 'pu', + presenceRequest: 'pr' +}; diff --git a/src/milestone-db/index.ts b/src/milestone-db/index.ts new file mode 100644 index 000000000..2f87005dd --- /dev/null +++ b/src/milestone-db/index.ts @@ -0,0 +1,84 @@ +import emitter = require('../emitter'); +import ShareDBError = require('../error'); +import util = require('../util'); + +var ERROR_CODE = ShareDBError.CODES; + +export = MilestoneDB; + +class MilestoneDB { + interval; + + constructor(options) { + emitter.EventEmitter.call(this); + + // The interval at which milestone snapshots should be saved + this.interval = options && options.interval; + } + + close(callback) { + if (callback) util.nextTick(callback); + } + + /** + * Fetch a milestone snapshot from the database + * @param {string} collection - name of the snapshot's collection + * @param {string} id - ID of the snapshot to fetch + * @param {number} version - the desired version of the milestone snapshot. The database will return + * the most recent milestone snapshot whose version is equal to or less than the provided value + * @param {Function} callback - a callback to invoke once the snapshot has been fetched. Should have + * the signature (error, snapshot) => void; + */ + getMilestoneSnapshot(collection, id, version, callback) { + var error = new ShareDBError( + ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, + 'getMilestoneSnapshot MilestoneDB method unimplemented' + ); + this._callBackOrEmitError(error, callback); + } + + /** + * @param {string} collection - name of the snapshot's collection + * @param {Snapshot} snapshot - the milestone snapshot to save + * @param {Function} callback (optional) - a callback to invoke after the snapshot has been saved. + * Should have the signature (error) => void; + */ + saveMilestoneSnapshot(collection, snapshot, callback) { + var error = new ShareDBError( + ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, + 'saveMilestoneSnapshot MilestoneDB method unimplemented' + ); + this._callBackOrEmitError(error, callback); + } + + getMilestoneSnapshotAtOrBeforeTime(collection, id, timestamp, callback) { + var error = new ShareDBError( + ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, + 'getMilestoneSnapshotAtOrBeforeTime MilestoneDB method unimplemented' + ); + this._callBackOrEmitError(error, callback); + } + + getMilestoneSnapshotAtOrAfterTime(collection, id, timestamp, callback) { + var error = new ShareDBError( + ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, + 'getMilestoneSnapshotAtOrAfterTime MilestoneDB method unimplemented' + ); + this._callBackOrEmitError(error, callback); + } + + _isValidVersion(version) { + return util.isValidVersion(version); + } + + _isValidTimestamp(timestamp) { + return util.isValidTimestamp(timestamp); + } + + _callBackOrEmitError(error, callback) { + if (callback) return util.nextTick(callback, error); + this.emit('error', error); + } +} + +emitter.mixin(MilestoneDB); diff --git a/src/milestone-db/memory.ts b/src/milestone-db/memory.ts new file mode 100644 index 000000000..0d185cd0c --- /dev/null +++ b/src/milestone-db/memory.ts @@ -0,0 +1,144 @@ +import MilestoneDB = require('./index'); +import ShareDBError = require('../error'); +import util = require('../util'); + +var ERROR_CODE = ShareDBError.CODES; + +export = MemoryMilestoneDB; + +/** + * In-memory ShareDB milestone database + * + * Milestone snapshots exist to speed up Backend.fetchSnapshot by providing milestones + * on top of which fewer ops can be applied to reach a desired version of the document. + * This very concept relies on persistence, which means that an in-memory database like + * this is in no way appropriate for production use. + * + * The main purpose of this class is to provide a simple example of implementation, + * and for use in tests. + */ +class MemoryMilestoneDB extends MilestoneDB { + _milestoneSnapshots; + + constructor(options) { + super(options); + + // Map from collection name -> doc id -> array of milestone snapshots + this._milestoneSnapshots = Object.create(null); + } + + getMilestoneSnapshot(collection, id, version, callback) { + if (!this._isValidVersion(version)) { + return util.nextTick(callback, new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Invalid version')); + } + + var predicate = versionLessThanOrEqualTo(version); + this._findMilestoneSnapshot(collection, id, predicate, callback); + } + + saveMilestoneSnapshot(collection, snapshot, callback) { + callback = callback || function(error) { + if (error) return this.emit('error', error); + this.emit('save', collection, snapshot); + }.bind(this); + + if (!collection) return callback(new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Missing collection')); + if (!snapshot) return callback(new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Missing snapshot')); + + var milestoneSnapshots = this._getMilestoneSnapshotsSync(collection, snapshot.id); + milestoneSnapshots.push(snapshot); + milestoneSnapshots.sort(function(a, b) { + return a.v - b.v; + }); + + util.nextTick(callback, null); + } + + getMilestoneSnapshotAtOrBeforeTime(collection, id, timestamp, callback) { + if (!this._isValidTimestamp(timestamp)) { + return util.nextTick(callback, new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Invalid timestamp')); + } + + var filter = timestampLessThanOrEqualTo(timestamp); + this._findMilestoneSnapshot(collection, id, filter, callback); + } + + getMilestoneSnapshotAtOrAfterTime(collection, id, timestamp, callback) { + if (!this._isValidTimestamp(timestamp)) { + return util.nextTick(callback, new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Invalid timestamp')); + } + + var filter = timestampGreaterThanOrEqualTo(timestamp); + this._findMilestoneSnapshot(collection, id, filter, function(error, snapshot) { + if (error) return util.nextTick(callback, error); + + var mtime = snapshot && snapshot.m && snapshot.m.mtime; + if (timestamp !== null && mtime < timestamp) { + snapshot = undefined; + } + + util.nextTick(callback, null, snapshot); + }); + } + + _findMilestoneSnapshot(collection, id, breakCondition, callback) { + if (!collection) { + return util.nextTick( + callback, new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Missing collection') + ); + } + if (!id) return util.nextTick(callback, new ShareDBError(ERROR_CODE.ERR_MILESTONE_ARGUMENT_INVALID, 'Missing ID')); + + var milestoneSnapshots = this._getMilestoneSnapshotsSync(collection, id); + + var milestoneSnapshot; + for (var i = 0; i < milestoneSnapshots.length; i++) { + var nextMilestoneSnapshot = milestoneSnapshots[i]; + if (breakCondition(milestoneSnapshot, nextMilestoneSnapshot)) { + break; + } else { + milestoneSnapshot = nextMilestoneSnapshot; + } + } + + util.nextTick(callback, null, milestoneSnapshot); + } + + _getMilestoneSnapshotsSync(collection, id) { + var collectionSnapshots = this._milestoneSnapshots[collection] || + (this._milestoneSnapshots[collection] = Object.create(null)); + return collectionSnapshots[id] || (collectionSnapshots[id] = []); + } +} + +function versionLessThanOrEqualTo(version) { + return function(currentSnapshot, nextSnapshot) { + if (version === null) { + return false; + } + + return nextSnapshot.v > version; + }; +} + +function timestampGreaterThanOrEqualTo(timestamp) { + return function(currentSnapshot) { + if (timestamp === null) { + return false; + } + + var mtime = currentSnapshot && currentSnapshot.m && currentSnapshot.m.mtime; + return mtime >= timestamp; + }; +} + +function timestampLessThanOrEqualTo(timestamp) { + return function(currentSnapshot, nextSnapshot) { + if (timestamp === null) { + return !!currentSnapshot; + } + + var mtime = nextSnapshot && nextSnapshot.m && nextSnapshot.m.mtime; + return mtime > timestamp; + }; +} diff --git a/src/milestone-db/no-op.ts b/src/milestone-db/no-op.ts new file mode 100644 index 000000000..f02ce87a3 --- /dev/null +++ b/src/milestone-db/no-op.ts @@ -0,0 +1,36 @@ +import MilestoneDB = require('./index'); +import util = require('../util'); + +export = NoOpMilestoneDB; + +/** + * A no-op implementation of the MilestoneDB class. + * + * This class exists as a simple, silent default drop-in for ShareDB, which allows the backend to call its methods with + * no effect. + */ +class NoOpMilestoneDB extends MilestoneDB { + constructor(options) { + super(options); + } + + getMilestoneSnapshot(collection, id, version, callback) { + var snapshot = undefined; + util.nextTick(callback, null, snapshot); + } + + saveMilestoneSnapshot(collection, snapshot, callback) { + if (callback) return util.nextTick(callback, null); + this.emit('save', collection, snapshot); + } + + getMilestoneSnapshotAtOrBeforeTime(collection, id, timestamp, callback) { + var snapshot = undefined; + util.nextTick(callback, null, snapshot); + } + + getMilestoneSnapshotAtOrAfterTime(collection, id, timestamp, callback) { + var snapshot = undefined; + util.nextTick(callback, null, snapshot); + } +} diff --git a/lib/next-tick.js b/src/next-tick.ts similarity index 79% rename from lib/next-tick.js rename to src/next-tick.ts index 659ed4df0..69feaf9a3 100644 --- a/lib/next-tick.js +++ b/src/next-tick.ts @@ -1,26 +1,26 @@ -exports.messageChannel = function() { - var triggerCallback = createNextTickTrigger(arguments); - var channel = new MessageChannel(); - channel.port1.onmessage = function() { - triggerCallback(); - channel.port1.close(); - }; - channel.port2.postMessage(''); -}; - -exports.setTimeout = function() { - var triggerCallback = createNextTickTrigger(arguments); - setTimeout(triggerCallback); -}; - -function createNextTickTrigger(args) { - var callback = args[0]; - var _args = []; - for (var i = 1; i < args.length; i++) { - _args[i - 1] = args[i]; - } - - return function triggerCallback() { - callback.apply(null, _args); - }; -} +export function messageChannel() { + var triggerCallback = createNextTickTrigger(arguments); + var channel = new MessageChannel(); + channel.port1.onmessage = function() { + triggerCallback(); + channel.port1.close(); + }; + channel.port2.postMessage(''); +} + +export function setTimeout() { + var triggerCallback = createNextTickTrigger(arguments); + global.setTimeout(triggerCallback); +} + +function createNextTickTrigger(args) { + var callback = args[0]; + var _args = []; + for (var i = 1; i < args.length; i++) { + _args[i - 1] = args[i]; + } + + return function triggerCallback() { + callback.apply(null, _args); + }; +} diff --git a/src/op-stream.ts b/src/op-stream.ts new file mode 100644 index 000000000..f57017f68 --- /dev/null +++ b/src/op-stream.ts @@ -0,0 +1,39 @@ +import { Readable } from 'stream'; +import util = require('./util'); + +// Stream of operations. Subscribe returns one of these +class OpStream extends Readable { + id; + open; + + constructor() { + super({objectMode: true}); + this.id = null; + this.open = true; + } + + pushData(data) { + // Ignore any messages after unsubscribe + if (!this.open) return; + // This data gets consumed in Agent#_subscribeToStream + this.push(data); + } + + destroy() { + // Only close stream once + if (!this.open) return; + this.open = false; + + this.push(null); + this.emit('close'); + } +} + +export = OpStream; + +// This function is for notifying us that the stream is empty and needs data. +// For now, we'll just ignore the signal and assume the reader reads as fast +// as we fill it. I could add a buffer in this function, but really I don't +// think that is any better than the buffer implementation in nodejs streams +// themselves. +OpStream.prototype._read = util.doNothing; diff --git a/lib/ot.js b/src/ot.ts similarity index 93% rename from lib/ot.js rename to src/ot.ts index 5cf349f60..0c6406b97 100644 --- a/lib/ot.js +++ b/src/ot.ts @@ -1,264 +1,265 @@ -// This contains the master OT functions for the database. They look like -// ot-types style operational transform functions, but they're a bit different. -// These functions understand versions and can deal with out of bound create & -// delete operations. - -var types = require('./types'); -var ShareDBError = require('./error'); -var util = require('./util'); - -var ERROR_CODE = ShareDBError.CODES; - -// Returns an error string on failure. Rockin' it C style. -exports.checkOp = function(op) { - if (op == null || typeof op !== 'object') { - return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'Op must be an object'); - } - - if (op.create != null) { - if (typeof op.create !== 'object') { - return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'Create data must be an object'); - } - var typeName = op.create.type; - if (typeof typeName !== 'string') { - return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'Missing create type'); - } - var type = types.map[typeName]; - if (type == null || typeof type !== 'object') { - return new ShareDBError(ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, 'Unknown type'); - } - } else if (op.del != null) { - if (op.del !== true) return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'del value must be true'); - } else if (!('op' in op)) { - return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'Missing op, create, or del'); - } - - if (op.src != null && typeof op.src !== 'string') { - return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'src must be a string'); - } - if (op.seq != null && typeof op.seq !== 'number') { - return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'seq must be a number'); - } - if ( - (op.src == null && op.seq != null) || - (op.src != null && op.seq == null) - ) { - return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'Both src and seq must be set together'); - } - - if (op.m != null && typeof op.m !== 'object') { - return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'op.m must be an object or null'); - } -}; - -// Takes in a string (type name or URI) and returns the normalized name (uri) -exports.normalizeType = function(typeName) { - return types.map[typeName] && types.map[typeName].uri; -}; - -// This is the super apply function that takes in snapshot data (including the -// type) and edits it in-place. Returns an error or null for success. -exports.apply = function(snapshot, op) { - if (typeof snapshot !== 'object') { - return new ShareDBError(ERROR_CODE.ERR_APPLY_SNAPSHOT_NOT_PROVIDED, 'Missing snapshot'); - } - if (snapshot.v != null && op.v != null && snapshot.v !== op.v) { - return new ShareDBError(ERROR_CODE.ERR_APPLY_OP_VERSION_DOES_NOT_MATCH_SNAPSHOT, 'Version mismatch'); - } - - // Create operation - if (op.create) { - if (snapshot.type) return new ShareDBError(ERROR_CODE.ERR_DOC_ALREADY_CREATED, 'Document already exists'); - - // The document doesn't exist, although it might have once existed - var create = op.create; - var type = types.map[create.type]; - if (!type) return new ShareDBError(ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, 'Unknown type'); - - try { - snapshot.data = type.create(create.data); - snapshot.type = type.uri; - snapshot.v++; - } catch (err) { - return err; - } - - // Delete operation - } else if (op.del) { - snapshot.data = undefined; - snapshot.type = null; - snapshot.v++; - - // Edit operation - } else if ('op' in op) { - var err = applyOpEdit(snapshot, op.op); - if (err) return err; - snapshot.v++; - - // No-op, and we don't have to do anything - } else { - snapshot.v++; - } -}; - -function applyOpEdit(snapshot, edit) { - if (!snapshot.type) return new ShareDBError(ERROR_CODE.ERR_DOC_DOES_NOT_EXIST, 'Document does not exist'); - - if (edit === undefined) return new ShareDBError(ERROR_CODE.ERR_OT_OP_NOT_PROVIDED, 'Missing op'); - var type = types.map[snapshot.type]; - if (!type) return new ShareDBError(ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, 'Unknown type'); - - if (type.name === 'json0' && Array.isArray(edit)) { - for (var i = 0; i < edit.length; i++) { - var opComponent = edit[i]; - if (Array.isArray(opComponent.p)) { - for (var j = 0; j < opComponent.p.length; j++) { - var pathSegment = opComponent.p[j]; - if (util.isDangerousProperty(pathSegment)) { - return new ShareDBError(ERROR_CODE.ERR_OT_OP_NOT_APPLIED, 'Invalid path segment'); - } - } - } - } - } - - try { - snapshot.data = type.apply(snapshot.data, edit); - } catch (err) { - return new ShareDBError(ERROR_CODE.ERR_OT_OP_NOT_APPLIED, err.message); - } -} - -exports.transform = function(type, op, appliedOp) { - // There are 16 cases this function needs to deal with - which are all the - // combinations of create/delete/op/noop from both op and appliedOp - if (op.v != null && op.v !== appliedOp.v) { - return new ShareDBError(ERROR_CODE.ERR_OP_VERSION_MISMATCH_DURING_TRANSFORM, 'Version mismatch'); - } - - if (appliedOp.del) { - if (op.create || 'op' in op) { - return new ShareDBError(ERROR_CODE.ERR_DOC_WAS_DELETED, 'Document was deleted'); - } - } else if ( - (appliedOp.create && ('op' in op || op.create || op.del)) || - ('op' in appliedOp && op.create) - ) { - // If appliedOp.create is not true, appliedOp contains an op - which - // also means the document exists remotely. - return new ShareDBError(ERROR_CODE.ERR_DOC_ALREADY_CREATED, 'Document was created remotely'); - } else if ('op' in appliedOp && 'op' in op) { - // If we reach here, they both have a .op property. - if (!type) return new ShareDBError(ERROR_CODE.ERR_DOC_DOES_NOT_EXIST, 'Document does not exist'); - - if (typeof type === 'string') { - type = types.map[type]; - if (!type) return new ShareDBError(ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, 'Unknown type'); - } - - try { - op.op = type.transform(op.op, appliedOp.op, 'left'); - } catch (err) { - return err; - } - } - - if (op.v != null) op.v++; -}; - -/** - * Apply an array of ops to the provided snapshot. - * - * @param snapshot - a Snapshot object which will be mutated by the provided ops - * @param ops - an array of ops to apply to the snapshot - * @param options - options (currently for internal use only) - * @return an error object if applicable - */ -exports.applyOps = function(snapshot, ops, options) { - options = options || {}; - for (var index = 0; index < ops.length; index++) { - var op = ops[index]; - if (options._normalizeLegacyJson0Ops) { - try { - normalizeLegacyJson0Ops(snapshot, op); - } catch (error) { - return new ShareDBError( - ERROR_CODE.ERR_OT_LEGACY_JSON0_OP_CANNOT_BE_NORMALIZED, - 'Cannot normalize legacy json0 op' - ); - } - } - snapshot.v = op.v; - var error = exports.apply(snapshot, op); - if (error) return error; - } -}; - -exports.transformPresence = function(presence, op, isOwnOp) { - var opError = this.checkOp(op); - if (opError) return opError; - - var type = presence.t; - if (typeof type === 'string') { - type = types.map[type]; - } - if (!type) return {code: ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, message: 'Unknown type'}; - if (!util.supportsPresence(type)) { - return {code: ERROR_CODE.ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE, message: 'Type does not support presence'}; - } - - if (op.create || op.del) { - presence.p = null; - presence.v++; - return; - } - - try { - presence.p = presence.p === null ? - null : - type.transformPresence(presence.p, op.op, isOwnOp); - } catch (error) { - return {code: ERROR_CODE.ERR_PRESENCE_TRANSFORM_FAILED, message: error.message || error}; - } - - presence.v++; -}; - -/** - * json0 had a breaking change in https://github.com/ottypes/json0/pull/40 - * The change added stricter type checking, which breaks fetchSnapshot() - * when trying to rebuild a snapshot from old, committed ops that didn't - * have this stricter validation. This method fixes up legacy ops to - * pass the stricter validation - */ -function normalizeLegacyJson0Ops(snapshot, json0Op) { - if (snapshot.type !== types.defaultType.uri) return; - var components = json0Op.op; - if (!components) return; - var data = snapshot.data; - - // type.apply() makes no guarantees about mutating the original data, so - // we need to clone. However, we only need to apply() if we have multiple - // components, so avoid cloning if we don't have to. - if (components.length > 1) data = util.clone(data); - - for (var i = 0; i < components.length; i++) { - var component = components[i]; - if (typeof component.lm === 'string') component.lm = +component.lm; - var path = component.p; - var element = data; - for (var j = 0; j < path.length; j++) { - var key = path[j]; - // https://github.com/ottypes/json0/blob/73db17e86adc5d801951d1a69453b01382e66c7d/lib/json0.js#L21 - if (Object.prototype.toString.call(element) == '[object Array]') path[j] = +key; - // https://github.com/ottypes/json0/blob/73db17e86adc5d801951d1a69453b01382e66c7d/lib/json0.js#L32 - else if (element.constructor === Object) path[j] = key.toString(); - element = element[key]; - } - - // Apply to update the snapshot, so we can correctly check the path for - // the next component. We don't need to do this on the final iteration, - // since there's no more ops. - if (i < components.length - 1) data = types.defaultType.apply(data, [component]); - } -} +// This contains the master OT functions for the database. They look like +// ot-types style operational transform functions, but they're a bit different. +// These functions understand versions and can deal with out of bound create & +// delete operations. + +import types = require('./types'); + +import ShareDBError = require('./error'); +import util = require('./util'); + +var ERROR_CODE = ShareDBError.CODES; + +// Returns an error string on failure. Rockin' it C style. +export function checkOp(op) { + if (op == null || typeof op !== 'object') { + return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'Op must be an object'); + } + + if (op.create != null) { + if (typeof op.create !== 'object') { + return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'Create data must be an object'); + } + var typeName = op.create.type; + if (typeof typeName !== 'string') { + return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'Missing create type'); + } + var type = types.map[typeName]; + if (type == null || typeof type !== 'object') { + return new ShareDBError(ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, 'Unknown type'); + } + } else if (op.del != null) { + if (op.del !== true) return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'del value must be true'); + } else if (!('op' in op)) { + return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'Missing op, create, or del'); + } + + if (op.src != null && typeof op.src !== 'string') { + return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'src must be a string'); + } + if (op.seq != null && typeof op.seq !== 'number') { + return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'seq must be a number'); + } + if ( + (op.src == null && op.seq != null) || + (op.src != null && op.seq == null) + ) { + return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'Both src and seq must be set together'); + } + + if (op.m != null && typeof op.m !== 'object') { + return new ShareDBError(ERROR_CODE.ERR_OT_OP_BADLY_FORMED, 'op.m must be an object or null'); + } +} + +// Takes in a string (type name or URI) and returns the normalized name (uri) +export function normalizeType(typeName) { + return types.map[typeName] && types.map[typeName].uri; +} + +// This is the super apply function that takes in snapshot data (including the +// type) and edits it in-place. Returns an error or null for success. +export function apply(snapshot, op) { + if (typeof snapshot !== 'object') { + return new ShareDBError(ERROR_CODE.ERR_APPLY_SNAPSHOT_NOT_PROVIDED, 'Missing snapshot'); + } + if (snapshot.v != null && op.v != null && snapshot.v !== op.v) { + return new ShareDBError(ERROR_CODE.ERR_APPLY_OP_VERSION_DOES_NOT_MATCH_SNAPSHOT, 'Version mismatch'); + } + + // Create operation + if (op.create) { + if (snapshot.type) return new ShareDBError(ERROR_CODE.ERR_DOC_ALREADY_CREATED, 'Document already exists'); + + // The document doesn't exist, although it might have once existed + var create = op.create; + var type = types.map[create.type]; + if (!type) return new ShareDBError(ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, 'Unknown type'); + + try { + snapshot.data = type.create(create.data); + snapshot.type = type.uri; + snapshot.v++; + } catch (err) { + return err; + } + + // Delete operation + } else if (op.del) { + snapshot.data = undefined; + snapshot.type = null; + snapshot.v++; + + // Edit operation + } else if ('op' in op) { + var err = applyOpEdit(snapshot, op.op); + if (err) return err; + snapshot.v++; + + // No-op, and we don't have to do anything + } else { + snapshot.v++; + } +} + +function applyOpEdit(snapshot, edit) { + if (!snapshot.type) return new ShareDBError(ERROR_CODE.ERR_DOC_DOES_NOT_EXIST, 'Document does not exist'); + + if (edit === undefined) return new ShareDBError(ERROR_CODE.ERR_OT_OP_NOT_PROVIDED, 'Missing op'); + var type = types.map[snapshot.type]; + if (!type) return new ShareDBError(ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, 'Unknown type'); + + if (type.name === 'json0' && Array.isArray(edit)) { + for (var i = 0; i < edit.length; i++) { + var opComponent = edit[i]; + if (Array.isArray(opComponent.p)) { + for (var j = 0; j < opComponent.p.length; j++) { + var pathSegment = opComponent.p[j]; + if (util.isDangerousProperty(pathSegment)) { + return new ShareDBError(ERROR_CODE.ERR_OT_OP_NOT_APPLIED, 'Invalid path segment'); + } + } + } + } + } + + try { + snapshot.data = type.apply(snapshot.data, edit); + } catch (err) { + return new ShareDBError(ERROR_CODE.ERR_OT_OP_NOT_APPLIED, err.message); + } +} + +export function transform(type, op, appliedOp) { + // There are 16 cases this function needs to deal with - which are all the + // combinations of create/delete/op/noop from both op and appliedOp + if (op.v != null && op.v !== appliedOp.v) { + return new ShareDBError(ERROR_CODE.ERR_OP_VERSION_MISMATCH_DURING_TRANSFORM, 'Version mismatch'); + } + + if (appliedOp.del) { + if (op.create || 'op' in op) { + return new ShareDBError(ERROR_CODE.ERR_DOC_WAS_DELETED, 'Document was deleted'); + } + } else if ( + (appliedOp.create && ('op' in op || op.create || op.del)) || + ('op' in appliedOp && op.create) + ) { + // If appliedOp.create is not true, appliedOp contains an op - which + // also means the document exists remotely. + return new ShareDBError(ERROR_CODE.ERR_DOC_ALREADY_CREATED, 'Document was created remotely'); + } else if ('op' in appliedOp && 'op' in op) { + // If we reach here, they both have a .op property. + if (!type) return new ShareDBError(ERROR_CODE.ERR_DOC_DOES_NOT_EXIST, 'Document does not exist'); + + if (typeof type === 'string') { + type = types.map[type]; + if (!type) return new ShareDBError(ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, 'Unknown type'); + } + + try { + op.op = type.transform(op.op, appliedOp.op, 'left'); + } catch (err) { + return err; + } + } + + if (op.v != null) op.v++; +} + +/** + * Apply an array of ops to the provided snapshot. + * + * @param snapshot - a Snapshot object which will be mutated by the provided ops + * @param ops - an array of ops to apply to the snapshot + * @param options - options (currently for internal use only) + * @return an error object if applicable + */ +export function applyOps(snapshot, ops, options) { + options = options || {}; + for (var index = 0; index < ops.length; index++) { + var op = ops[index]; + if (options._normalizeLegacyJson0Ops) { + try { + normalizeLegacyJson0Ops(snapshot, op); + } catch (error) { + return new ShareDBError( + ERROR_CODE.ERR_OT_LEGACY_JSON0_OP_CANNOT_BE_NORMALIZED, + 'Cannot normalize legacy json0 op' + ); + } + } + snapshot.v = op.v; + var error = exports.apply(snapshot, op); + if (error) return error; + } +} + +export function transformPresence(presence, op, isOwnOp) { + var opError = this.checkOp(op); + if (opError) return opError; + + var type = presence.t; + if (typeof type === 'string') { + type = types.map[type]; + } + if (!type) return {code: ERROR_CODE.ERR_DOC_TYPE_NOT_RECOGNIZED, message: 'Unknown type'}; + if (!util.supportsPresence(type)) { + return {code: ERROR_CODE.ERR_TYPE_DOES_NOT_SUPPORT_PRESENCE, message: 'Type does not support presence'}; + } + + if (op.create || op.del) { + presence.p = null; + presence.v++; + return; + } + + try { + presence.p = presence.p === null ? + null : + type.transformPresence(presence.p, op.op, isOwnOp); + } catch (error) { + return {code: ERROR_CODE.ERR_PRESENCE_TRANSFORM_FAILED, message: error.message || error}; + } + + presence.v++; +} + +/** + * json0 had a breaking change in https://github.com/ottypes/json0/pull/40 + * The change added stricter type checking, which breaks fetchSnapshot() + * when trying to rebuild a snapshot from old, committed ops that didn't + * have this stricter validation. This method fixes up legacy ops to + * pass the stricter validation + */ +function normalizeLegacyJson0Ops(snapshot, json0Op) { + if (snapshot.type !== types.defaultType.uri) return; + var components = json0Op.op; + if (!components) return; + var data = snapshot.data; + + // type.apply() makes no guarantees about mutating the original data, so + // we need to clone. However, we only need to apply() if we have multiple + // components, so avoid cloning if we don't have to. + if (components.length > 1) data = util.clone(data); + + for (var i = 0; i < components.length; i++) { + var component = components[i]; + if (typeof component.lm === 'string') component.lm = +component.lm; + var path = component.p; + var element = data; + for (var j = 0; j < path.length; j++) { + var key = path[j]; + // https://github.com/ottypes/json0/blob/73db17e86adc5d801951d1a69453b01382e66c7d/lib/json0.js#L21 + if (Object.prototype.toString.call(element) == '[object Array]') path[j] = +key; + // https://github.com/ottypes/json0/blob/73db17e86adc5d801951d1a69453b01382e66c7d/lib/json0.js#L32 + else if (element.constructor === Object) path[j] = key.toString(); + element = element[key]; + } + + // Apply to update the snapshot, so we can correctly check the path for + // the next component. We don't need to do this on the final iteration, + // since there's no more ops. + if (i < components.length - 1) data = types.defaultType.apply(data, [component]); + } +} diff --git a/lib/projections.js b/src/projections.ts similarity index 88% rename from lib/projections.js rename to src/projections.ts index b9a43c201..ff515c5c5 100644 --- a/lib/projections.js +++ b/src/projections.ts @@ -1,128 +1,128 @@ -var json0 = require('ot-json0').type; -var ShareDBError = require('./error'); -var util = require('./util'); - -var ERROR_CODE = ShareDBError.CODES; - -exports.projectSnapshot = projectSnapshot; -exports.projectSnapshots = projectSnapshots; -exports.projectOp = projectOp; -exports.isSnapshotAllowed = isSnapshotAllowed; -exports.isOpAllowed = isOpAllowed; - - -// Project a snapshot in place to only include specified fields -function projectSnapshot(fields, snapshot) { - // Only json0 supported right now - if (snapshot.type && snapshot.type !== json0.uri) { - throw new Error(ERROR_CODE.ERR_TYPE_CANNOT_BE_PROJECTED, 'Cannot project snapshots of type ' + snapshot.type); - } - snapshot.data = projectData(fields, snapshot.data); -} - -function projectSnapshots(fields, snapshots) { - for (var i = 0; i < snapshots.length; i++) { - var snapshot = snapshots[i]; - projectSnapshot(fields, snapshot); - } -} - -function projectOp(fields, op) { - if (op.create) { - projectSnapshot(fields, op.create); - } - if ('op' in op) { - op.op = projectEdit(fields, op.op); - } -} - -function projectEdit(fields, op) { - // So, we know the op is a JSON op - var result = []; - - for (var i = 0; i < op.length; i++) { - var c = op[i]; - var path = c.p; - - if (path.length === 0) { - var newC = {p: []}; - - if (c.od !== undefined || c.oi !== undefined) { - if (c.od !== undefined) { - newC.od = projectData(fields, c.od); - } - if (c.oi !== undefined) { - newC.oi = projectData(fields, c.oi); - } - result.push(newC); - } - } else { - // The path has a first element. Just check it against the fields. - if (fields[path[0]]) { - result.push(c); - } - } - } - return result; -} - -function isOpAllowed(knownType, fields, op) { - if (op.create) { - return isSnapshotAllowed(fields, op.create); - } - if ('op' in op) { - if (knownType && knownType !== json0.uri) return false; - return isEditAllowed(fields, op.op); - } - // Noop and del are both ok. - return true; -} - -// Basically, would the projected version of this data be the same as the original? -function isSnapshotAllowed(fields, snapshot) { - if (snapshot.type && snapshot.type !== json0.uri) { - return false; - } - if (snapshot.data == null) { - return true; - } - // Data must be an object if not null - if (typeof snapshot.data !== 'object' || Array.isArray(snapshot.data)) { - return false; - } - for (var k in snapshot.data) { - if (!fields[k]) return false; - } - return true; -} - -function isEditAllowed(fields, op) { - for (var i = 0; i < op.length; i++) { - var c = op[i]; - if (c.p.length === 0) { - return false; - } else if (!fields[c.p[0]]) { - return false; - } - } - return true; -} - -function projectData(fields, data) { - // Return back null or undefined - if (data == null) { - return data; - } - // If data is not an object, the projected version just looks like null. - if (typeof data !== 'object' || Array.isArray(data)) { - return null; - } - // Shallow copy of each field - var result = {}; - for (var key in fields) { - if (util.hasOwn(data, key)) { - result[key] = data[key]; - } - } - return result; -} +import { type as json0 } from 'ot-json0'; +import ShareDBError = require('./error'); +import util = require('./util'); + +var ERROR_CODE = ShareDBError.CODES; + +export { projectSnapshot }; +export { projectSnapshots }; +export { projectOp }; +export { isSnapshotAllowed }; +export { isOpAllowed }; + + +// Project a snapshot in place to only include specified fields +function projectSnapshot(fields, snapshot) { + // Only json0 supported right now + if (snapshot.type && snapshot.type !== json0.uri) { + throw new Error(ERROR_CODE.ERR_TYPE_CANNOT_BE_PROJECTED, 'Cannot project snapshots of type ' + snapshot.type); + } + snapshot.data = projectData(fields, snapshot.data); +} + +function projectSnapshots(fields, snapshots) { + for (var i = 0; i < snapshots.length; i++) { + var snapshot = snapshots[i]; + projectSnapshot(fields, snapshot); + } +} + +function projectOp(fields, op) { + if (op.create) { + projectSnapshot(fields, op.create); + } + if ('op' in op) { + op.op = projectEdit(fields, op.op); + } +} + +function projectEdit(fields, op) { + // So, we know the op is a JSON op + var result = []; + + for (var i = 0; i < op.length; i++) { + var c = op[i]; + var path = c.p; + + if (path.length === 0) { + var newC = {p: []}; + + if (c.od !== undefined || c.oi !== undefined) { + if (c.od !== undefined) { + newC.od = projectData(fields, c.od); + } + if (c.oi !== undefined) { + newC.oi = projectData(fields, c.oi); + } + result.push(newC); + } + } else { + // The path has a first element. Just check it against the fields. + if (fields[path[0]]) { + result.push(c); + } + } + } + return result; +} + +function isOpAllowed(knownType, fields, op) { + if (op.create) { + return isSnapshotAllowed(fields, op.create); + } + if ('op' in op) { + if (knownType && knownType !== json0.uri) return false; + return isEditAllowed(fields, op.op); + } + // Noop and del are both ok. + return true; +} + +// Basically, would the projected version of this data be the same as the original? +function isSnapshotAllowed(fields, snapshot) { + if (snapshot.type && snapshot.type !== json0.uri) { + return false; + } + if (snapshot.data == null) { + return true; + } + // Data must be an object if not null + if (typeof snapshot.data !== 'object' || Array.isArray(snapshot.data)) { + return false; + } + for (var k in snapshot.data) { + if (!fields[k]) return false; + } + return true; +} + +function isEditAllowed(fields, op) { + for (var i = 0; i < op.length; i++) { + var c = op[i]; + if (c.p.length === 0) { + return false; + } else if (!fields[c.p[0]]) { + return false; + } + } + return true; +} + +function projectData(fields, data) { + // Return back null or undefined + if (data == null) { + return data; + } + // If data is not an object, the projected version just looks like null. + if (typeof data !== 'object' || Array.isArray(data)) { + return null; + } + // Shallow copy of each field + var result = {}; + for (var key in fields) { + if (util.hasOwn(data, key)) { + result[key] = data[key]; + } + } + return result; +} diff --git a/lib/protocol.js b/src/protocol.ts similarity index 94% rename from lib/protocol.js rename to src/protocol.ts index af78278a3..60139cab6 100644 --- a/lib/protocol.js +++ b/src/protocol.ts @@ -1,28 +1,28 @@ -module.exports = { - major: 1, - minor: 2, - checkAtLeast: checkAtLeast -}; - -function checkAtLeast(toCheck, checkAgainst) { - toCheck = normalizedProtocol(toCheck); - checkAgainst = normalizedProtocol(checkAgainst); - if (toCheck.major > checkAgainst.major) return true; - return toCheck.major === checkAgainst.major && - toCheck.minor >= checkAgainst.minor; -} - -function normalizedProtocol(protocol) { - if (typeof protocol === 'string') { - var segments = protocol.split('.'); - protocol = { - major: segments[0], - minor: segments[1] - }; - } - - return { - major: +(protocol.protocol || protocol.major || 0), - minor: +(protocol.protocolMinor || protocol.minor || 0) - }; -} +export = { + major: 1, + minor: 2, + checkAtLeast: checkAtLeast +}; + +function checkAtLeast(toCheck, checkAgainst) { + toCheck = normalizedProtocol(toCheck); + checkAgainst = normalizedProtocol(checkAgainst); + if (toCheck.major > checkAgainst.major) return true; + return toCheck.major === checkAgainst.major && + toCheck.minor >= checkAgainst.minor; +} + +function normalizedProtocol(protocol) { + if (typeof protocol === 'string') { + var segments = protocol.split('.'); + protocol = { + major: segments[0], + minor: segments[1] + }; + } + + return { + major: +(protocol.protocol || protocol.major || 0), + minor: +(protocol.protocolMinor || protocol.minor || 0) + }; +} diff --git a/src/pubsub/index.ts b/src/pubsub/index.ts new file mode 100644 index 000000000..f8426fe9c --- /dev/null +++ b/src/pubsub/index.ts @@ -0,0 +1,148 @@ +import emitter = require('../emitter'); +import OpStream = require('../op-stream'); +import ShareDBError = require('../error'); +import util = require('../util'); + +var ERROR_CODE = ShareDBError.CODES; + +class PubSub { + prefix; + nextStreamId; + streamsCount; + streams; + subscribed; + _defaultCallback; + + constructor(options) { + if (!(this instanceof PubSub)) return new PubSub(options); + emitter.EventEmitter.call(this); + + this.prefix = options && options.prefix; + this.nextStreamId = 1; + this.streamsCount = 0; + // Maps channel -> id -> stream + this.streams = Object.create(null); + // State for tracking subscriptions. We track this.subscribed separately from + // the streams, since the stream gets added synchronously, and the subscribe + // isn't complete until the callback returns from Redis + // Maps channel -> true + this.subscribed = Object.create(null); + + var pubsub = this; + this._defaultCallback = function(err) { + if (err) return pubsub.emit('error', err); + }; + } + + close(callback) { + for (var channel in this.streams) { + var map = this.streams[channel]; + for (var id in map) { + map[id].destroy(); + } + } + if (callback) util.nextTick(callback); + } + + _subscribe(channel, callback) { + util.nextTick(function() { + callback(new ShareDBError( + ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, + '_subscribe PubSub method unimplemented' + )); + }); + } + + _unsubscribe(channel, callback) { + util.nextTick(function() { + callback(new ShareDBError( + ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, + '_unsubscribe PubSub method unimplemented' + )); + }); + } + + _publish(channels, data, callback) { + util.nextTick(function() { + callback(new ShareDBError(ERROR_CODE.ERR_DATABASE_METHOD_NOT_IMPLEMENTED, '_publish PubSub method unimplemented')); + }); + } + + subscribe(channel, callback) { + if (!callback) callback = this._defaultCallback; + if (this.prefix) { + channel = this.prefix + ' ' + channel; + } + + var pubsub = this; + if (this.subscribed[channel]) { + util.nextTick(function() { + var stream = pubsub._createStream(channel); + callback(null, stream); + }); + return; + } + this._subscribe(channel, function(err) { + if (err) return callback(err); + pubsub.subscribed[channel] = true; + var stream = pubsub._createStream(channel); + callback(null, stream); + }); + } + + publish(channels, data, callback) { + if (!callback) callback = this._defaultCallback; + if (this.prefix) { + for (var i = 0; i < channels.length; i++) { + channels[i] = this.prefix + ' ' + channels[i]; + } + } + this._publish(channels, data, callback); + } + + _emit(channel, data) { + var channelStreams = this.streams[channel]; + if (channelStreams) { + for (var id in channelStreams) { + channelStreams[id].pushData(data); + } + } + } + + _createStream(channel) { + var stream = new OpStream(); + var pubsub = this; + stream.once('close', function() { + pubsub._removeStream(channel, stream); + }); + + this.streamsCount++; + var map = this.streams[channel] || (this.streams[channel] = Object.create(null)); + stream.id = this.nextStreamId++; + map[stream.id] = stream; + + return stream; + } + + _removeStream(channel, stream) { + var map = this.streams[channel]; + if (!map) return; + + this.streamsCount--; + delete map[stream.id]; + + // Cleanup if this was the last subscribed stream for the channel + if (util.hasKeys(map)) return; + delete this.streams[channel]; + // Synchronously clear subscribed state. We won't actually be unsubscribed + // until some unknown time in the future. If subscribe is called in this + // period, we want to send a subscription message and wait for it to + // complete before we can count on being subscribed again + delete this.subscribed[channel]; + + this._unsubscribe(channel, this._defaultCallback); + } +} + +export = PubSub; +emitter.mixin(PubSub); diff --git a/src/pubsub/memory.ts b/src/pubsub/memory.ts new file mode 100644 index 000000000..69485efd7 --- /dev/null +++ b/src/pubsub/memory.ts @@ -0,0 +1,39 @@ +import PubSub = require('./index'); +import util = require('../util'); + +// In-memory ShareDB pub/sub +// +// This is a fully functional implementation. Since ShareDB does not require +// persistence of pub/sub state, it may be used in production environments +// requiring only a single stand alone server process. Additionally, it is +// easy to swap in an external pub/sub adapter if/when additional server +// processes are desired. No pub/sub APIs are adapter specific. + +class MemoryPubSub extends PubSub { + constructor(options) { + super(options); + } + + _subscribe(channel, callback) { + util.nextTick(callback); + } + + _unsubscribe(channel, callback) { + util.nextTick(callback); + } + + _publish(channels, data, callback) { + var pubsub = this; + util.nextTick(function() { + for (var i = 0; i < channels.length; i++) { + var channel = channels[i]; + if (pubsub.subscribed[channel]) { + pubsub._emit(channel, data); + } + } + callback(); + }); + } +} + +export = MemoryPubSub; diff --git a/src/query-emitter.ts b/src/query-emitter.ts new file mode 100644 index 000000000..b544656f9 --- /dev/null +++ b/src/query-emitter.ts @@ -0,0 +1,377 @@ +import arraydiff = require('arraydiff'); +import deepEqual = require('fast-deep-equal'); +import ShareDBError = require('./error'); +import util = require('./util'); + +var ERROR_CODE = ShareDBError.CODES; + +class QueryEmitter { + backend; + agent; + db; + index; + query; + collection; + fields; + options; + snapshotProjection; + streams; + ids; + extra; + skipPoll; + canPollDoc; + pollDebounce; + pollInterval; + _polling; + _pendingPoll; + _pollDebounceId; + _pollIntervalId; + + constructor(request, streams, ids, extra) { + this.backend = request.backend; + this.agent = request.agent; + this.db = request.db; + this.index = request.index; + this.query = request.query; + this.collection = request.collection; + this.fields = request.fields; + this.options = request.options; + this.snapshotProjection = request.snapshotProjection; + this.streams = streams; + this.ids = ids; + this.extra = extra; + + this.skipPoll = this.options.skipPoll || util.doNothing; + this.canPollDoc = this.db.canPollDoc(this.collection, this.query); + this.pollDebounce = + (typeof this.options.pollDebounce === 'number') ? this.options.pollDebounce : + (typeof this.db.pollDebounce === 'number') ? this.db.pollDebounce : + streams.length > 1 ? 1000 : 0; + this.pollInterval = + (typeof this.options.pollInterval === 'number') ? this.options.pollInterval : + (typeof this.db.pollInterval === 'number') ? this.db.pollInterval : + streams.length > 1 ? 1000 : 0; + + this._polling = false; + this._pendingPoll = null; + this._pollDebounceId = null; + this._pollIntervalId = null; + } + + // Start processing events from the stream + _open() { + var emitter = this; + this._defaultCallback = function(err) { + if (err) emitter.onError(err); + }; + + emitter.streams.forEach(function(stream) { + stream.on('data', function(data) { + if (data.error) { + return emitter.onError(data.error); + } + emitter._update(data); + }); + stream.on('end', function() { + emitter.destroy(); + }); + }); + + // Make sure we start polling if pollInterval is being used + this._flushPoll(); + } + + destroy() { + clearTimeout(this._pollDebounceId); + clearTimeout(this._pollIntervalId); + + var stream; + + while (stream = this.streams.pop()) { + stream.destroy(); + } + } + + _emitTiming(action, start) { + this.backend.emit('timing', action, Date.now() - start, this); + } + + _update(op) { + // Note that `op` should not be projected or sanitized yet. It's possible for + // a query to filter on a field that's not in the projection. skipPoll checks + // to see if an op could possibly affect a query, so it should get passed the + // full op. The onOp listener function must call backend.sanitizeOp() + var id = op.d; + var pollCallback = this._defaultCallback; + + // Check if the op's id matches the query before updating the query results + // and send it through immediately if it does. The current snapshot + // (including the op) for a newly matched document will get sent in the + // insert diff, so we don't need to send the op that caused the doc to + // match. If the doc already exists in the client and isn't otherwise + // subscribed, the client will need to request the op when it receives the + // snapshot from the query to bring itself up to date. + // + // The client may see the result of the op get reflected before the query + // results update. This might prove janky in some cases, since a doc could + // get deleted before it is removed from the results, for example. However, + // it will mean that ops which don't end up changing the results are + // received sooner even if query polling takes a while. + // + // Alternatively, we could send the op message only after the query has + // updated, and it would perhaps be ideal to send in the same message to + // avoid the user seeing transitional states where the doc is updated but + // the results order is not. + // + // We should send the op even if it is the op that causes the document to no + // longer match the query. If client-side filters are applied to the model + // to figure out which documents to render in a list, we will want the op + // that removed the doc from the query to cause the client-side computed + // list to update. + if (this.ids.indexOf(id) !== -1) { + var emitter = this; + pollCallback = function(err) { + // Send op regardless of polling error. Clients handle subscription to ops + // on the documents that currently match query results independently from + // updating which docs match the query + emitter.onOp(op); + if (err) emitter.onError(err); + }; + } + + // Ignore if the database or user function says we don't need to poll + try { + if ( + this.db.skipPoll(this.collection, id, op, this.query) || + this.skipPoll(this.collection, id, op, this.query) + ) { + return pollCallback(); + } + } catch (err) { + return pollCallback(err); + } + if (this.canPollDoc) { + // We can query against only the document that was modified to see if the + // op has changed whether or not it matches the results + this.queryPollDoc(id, pollCallback); + } else { + // We need to do a full poll of the query, because the query uses limits, + // sorts, or something special + this.queryPoll(pollCallback); + } + } + + _flushPoll() { + // Don't send another polling query at the same time or within the debounce + // timeout. This function will be called again once the poll that is + // currently in progress or the pollDebounce timeout completes + if (this._polling || this._pollDebounceId) return; + + // If another polling event happened while we were polling, call poll again, + // as the results may have changed + if (this._pendingPoll) { + this.queryPoll(); + + // If a pollInterval is specified, poll if the query doesn't get polled in + // the time of the interval + } else if (this.pollInterval) { + var emitter = this; + this._pollIntervalId = setTimeout(function() { + emitter._pollIntervalId = null; + emitter.queryPoll(emitter._defaultCallback); + }, this.pollInterval); + } + } + + queryPoll(callback) { + var emitter = this; + + // Only run a single polling check against mongo at a time per emitter. This + // matters for two reasons: First, one callback could return before the + // other. Thus, our result diffs could get out of order, and the clients + // could end up with results in a funky order and the wrong results being + // mutated in the query. Second, only having one query executed + // simultaneously per emitter will act as a natural adaptive rate limiting + // in case the db is under load. + // + // This isn't necessary for the document polling case, since they operate + // on a given id and won't accidentally modify the wrong doc. Also, those + // queries should be faster and are less likely to be the same, so there is + // less benefit to possible load reduction. + if (this._polling || this._pollDebounceId) { + if (this._pendingPoll) { + this._pendingPoll.push(callback); + } else { + this._pendingPoll = [callback]; + } + return; + } + this._polling = true; + var pending = this._pendingPoll; + this._pendingPoll = null; + if (this.pollDebounce) { + this._pollDebounceId = setTimeout(function() { + emitter._pollDebounceId = null; + emitter._flushPoll(); + }, this.pollDebounce); + } + clearTimeout(this._pollIntervalId); + + var start = Date.now(); + this.db.queryPoll(this.collection, this.query, this.options, function(err, ids, extra) { + if (err) return emitter._finishPoll(err, callback, pending); + emitter._emitTiming('queryEmitter.poll', start); + + // Be nice to not have to do this in such a brute force way + if (!deepEqual(emitter.extra, extra)) { + emitter.extra = extra; + emitter.onExtra(extra); + } + + var idsDiff = arraydiff(emitter.ids, ids); + if (idsDiff.length) { + emitter.ids = ids; + var inserted = getInserted(idsDiff); + if (inserted.length) { + var snapshotOptions = {}; + snapshotOptions.agentCustom = emitter.agent.custom; + + function _getSnapshotBulkCb(err, snapshotMap) { + if (err) return emitter._finishPoll(err, callback, pending); + var snapshots = emitter.backend._getSnapshotsFromMap(inserted, snapshotMap); + var snapshotType = emitter.backend.SNAPSHOT_TYPES.current; + emitter.backend._sanitizeSnapshots( + emitter.agent, + emitter.snapshotProjection, + emitter.collection, + snapshots, + snapshotType, + function(err) { + if (err) return emitter._finishPoll(err, callback, pending); + emitter._emitTiming('queryEmitter.pollGetSnapshotBulk', start); + var diff = mapDiff(idsDiff, snapshotMap); + emitter.onDiff(diff); + emitter._finishPoll(err, callback, pending); + }); + }; + emitter.db.getSnapshotBulk(emitter.collection, inserted, emitter.fields, snapshotOptions, _getSnapshotBulkCb); + } else { + emitter.onDiff(idsDiff); + emitter._finishPoll(err, callback, pending); + } + } else { + emitter._finishPoll(err, callback, pending); + } + }); + } + + _finishPoll(err, callback, pending) { + this._polling = false; + if (callback) callback(err); + if (pending) { + for (var i = 0; i < pending.length; i++) { + callback = pending[i]; + if (callback) callback(err); + } + } + this._flushPoll(); + } + + queryPollDoc(id, callback) { + var emitter = this; + var start = Date.now(); + this.db.queryPollDoc(this.collection, id, this.query, this.options, function(err, matches) { + if (err) return callback(err); + emitter._emitTiming('queryEmitter.pollDoc', start); + + // Check if the document was in the previous results set + var i = emitter.ids.indexOf(id); + + if (i === -1 && matches) { + // Add doc to the collection. Order isn't important, so we'll just whack + // it at the end + var index = emitter.ids.push(id) - 1; + + var snapshotOptions = {}; + snapshotOptions.agentCustom = emitter.agent.custom; + + // We can get the result to send to the client async, since there is a + // delay in sending to the client anyway + emitter.db.getSnapshot(emitter.collection, id, emitter.fields, snapshotOptions, function(err, snapshot) { + if (err) return callback(err); + var snapshots = [snapshot]; + var snapshotType = emitter.backend.SNAPSHOT_TYPES.current; + emitter.backend._sanitizeSnapshots( + emitter.agent, + emitter.snapshotProjection, + emitter.collection, + snapshots, + snapshotType, + function(err) { + if (err) return callback(err); + emitter.onDiff([new arraydiff.InsertDiff(index, snapshots)]); + emitter._emitTiming('queryEmitter.pollDocGetSnapshot', start); + callback(); + }); + }); + return; + } + + if (i !== -1 && !matches) { + emitter.ids.splice(i, 1); + emitter.onDiff([new arraydiff.RemoveDiff(i, 1)]); + return callback(); + } + + callback(); + }); + } +} + +export = QueryEmitter; + +// Clients must assign each of these functions synchronously after constructing +// an instance of QueryEmitter. The instance is subscribed to an op stream at +// construction time, and does not buffer emitted events. Diff events assume +// all messages are received and applied in order, so it is critical that none +// are dropped. +QueryEmitter.prototype.onError = + QueryEmitter.prototype.onDiff = + QueryEmitter.prototype.onExtra = + QueryEmitter.prototype.onOp = function() { + throw new ShareDBError( + ERROR_CODE.ERR_QUERY_EMITTER_LISTENER_NOT_ASSIGNED, + 'Required QueryEmitter listener not assigned' + ); + }; + +function getInserted(diff) { + var inserted = []; + for (var i = 0; i < diff.length; i++) { + var item = diff[i]; + if (item instanceof arraydiff.InsertDiff) { + for (var j = 0; j < item.values.length; j++) { + inserted.push(item.values[j]); + } + } + } + return inserted; +} + +function mapDiff(idsDiff, snapshotMap) { + var diff = []; + for (var i = 0; i < idsDiff.length; i++) { + var item = idsDiff[i]; + if (item instanceof arraydiff.InsertDiff) { + var values = []; + for (var j = 0; j < item.values.length; j++) { + var id = item.values[j]; + values.push(snapshotMap[id]); + } + diff.push(new arraydiff.InsertDiff(item.index, values)); + } else { + diff.push(item); + } + } + return diff; +} diff --git a/src/read-snapshots-request.ts b/src/read-snapshots-request.ts new file mode 100644 index 000000000..e4fedee83 --- /dev/null +++ b/src/read-snapshots-request.ts @@ -0,0 +1,119 @@ +import ShareDBError = require('./error'); + +export = ReadSnapshotsRequest; + +/** + * Context object passed to "readSnapshots" middleware functions + * + * @param {string} collection + * @param {Snapshot[]} snapshots - snapshots being read + * @param {keyof Backend.prototype.SNAPSHOT_TYPES} snapshotType - the type of snapshot read being + * performed + */ +class ReadSnapshotsRequest { + collection; + snapshots; + snapshotType; + action; + agent; + backend; + _idToError; + + constructor(collection, snapshots, snapshotType) { + this.collection = collection; + this.snapshots = snapshots; + this.snapshotType = snapshotType; + + // Added by Backend#trigger + this.action = null; + this.agent = null; + this.backend = null; + + /** + * Map of doc id to error: `{[docId: string]: string | Error}` + */ + this._idToError = null; + } + + /** + * Rejects the read of a specific snapshot. A rejected snapshot read will not have that snapshot's + * data sent down to the client. + * + * If the error has a `code` property of `"ERR_SNAPSHOT_READ_SILENT_REJECTION"`, then the Share + * client will not pass the error to user code, but will still do things like cancel subscriptions. + * The `#rejectSnapshotReadSilent(snapshot, errorMessage)` method can also be used for convenience. + * + * @param {Snapshot} snapshot + * @param {string | Error} error + * + * @see #rejectSnapshotReadSilent + * @see ShareDBError.CODES.ERR_SNAPSHOT_READ_SILENT_REJECTION + * @see ShareDBError.CODES.ERR_SNAPSHOT_READS_REJECTED + */ + rejectSnapshotRead(snapshot, error) { + if (!this._idToError) { + this._idToError = Object.create(null); + } + this._idToError[snapshot.id] = error; + } + + /** + * Rejects the read of a specific snapshot. A rejected snapshot read will not have that snapshot's + * data sent down to the client. + * + * This method will set a special error code that causes the Share client to not pass the error to + * user code, though it will still do things like cancel subscriptions. + * + * @param {Snapshot} snapshot + * @param {string} errorMessage + */ + rejectSnapshotReadSilent(snapshot, errorMessage) { + this.rejectSnapshotRead(snapshot, this.silentRejectionError(errorMessage)); + } + + silentRejectionError(errorMessage) { + return new ShareDBError(ShareDBError.CODES.ERR_SNAPSHOT_READ_SILENT_REJECTION, errorMessage); + } + + /** + * Returns whether this trigger of "readSnapshots" has had a snapshot read rejected. + */ + hasSnapshotRejection() { + return this._idToError != null; + } + + /** + * Returns an overall error from "readSnapshots" based on the snapshot-specific errors. + * + * - If there's exactly one snapshot and it has an error, then that error is returned. + * - If there's more than one snapshot and at least one has an error, then an overall + * "ERR_SNAPSHOT_READS_REJECTED" is returned, with an `idToError` property. + */ + getReadSnapshotsError() { + var snapshots = this.snapshots; + var idToError = this._idToError; + // If there are 0 snapshots, there can't be any snapshot-specific errors. + if (snapshots.length === 0) { + return; + } + + // Single snapshot with error is treated as a full error. + if (snapshots.length === 1) { + var snapshotError = idToError[snapshots[0].id]; + if (snapshotError) { + return snapshotError; + } else { + return; + } + } + + // Errors in specific snapshots result in an overall ERR_SNAPSHOT_READS_REJECTED. + // + // fetchBulk and subscribeBulk know how to handle that special error by sending a doc-by-doc + // success/failure to the client. Other methods that don't or can't handle partial failures + // will treat it as a full rejection. + var err = new ShareDBError(ShareDBError.CODES.ERR_SNAPSHOT_READS_REJECTED); + err.idToError = idToError; + return err; + } +} diff --git a/src/snapshot.ts b/src/snapshot.ts new file mode 100644 index 000000000..700417b6b --- /dev/null +++ b/src/snapshot.ts @@ -0,0 +1,17 @@ +export = Snapshot; + +class Snapshot { + id; + v; + type; + data; + m; + + constructor(id, version, type, data, meta) { + this.id = id; + this.v = version; + this.type = type; + this.data = data; + this.m = meta; + } +} diff --git a/src/stream-socket.ts b/src/stream-socket.ts new file mode 100644 index 000000000..a6a3c1040 --- /dev/null +++ b/src/stream-socket.ts @@ -0,0 +1,74 @@ +import { Duplex } from 'stream'; +import logger = require('./logger'); +import util = require('./util'); + +class StreamSocket { + readyState; + stream; + + constructor() { + this.readyState = 0; + this.stream = new ServerStream(this); + } + + _open() { + if (this.readyState !== 0) return; + this.readyState = 1; + this.onopen(); + } + + close(reason) { + if (this.readyState === 3) return; + this.readyState = 3; + // Signal data writing is complete. Emits the 'end' event + this.stream.push(null); + this.onclose(reason || 'closed'); + } + + send(data) { + // Data is an object + this.stream.push(JSON.parse(data)); + } +} + +export = StreamSocket; + +StreamSocket.prototype.onmessage = util.doNothing; +StreamSocket.prototype.onclose = util.doNothing; +StreamSocket.prototype.onerror = util.doNothing; +StreamSocket.prototype.onopen = util.doNothing; + + +class ServerStream extends Duplex { + socket; + + constructor(socket) { + super({objectMode: true}); + + this.socket = socket; + + this.on('error', function(error) { + logger.warn('ShareDB client message stream error', error); + socket.close('stopped'); + }); + + // The server ended the writable stream. Triggered by calling stream.end() + // in agent.close() + this.on('finish', function() { + socket.close('stopped'); + }); + } + + _write(chunk, encoding, callback) { + var socket = this.socket; + util.nextTick(function() { + if (socket.readyState !== 1) return; + socket.onmessage({data: JSON.stringify(chunk)}); + callback(); + }); + } +} + +ServerStream.prototype.isServer = true; + +ServerStream.prototype._read = util.doNothing; diff --git a/src/submit-request.ts b/src/submit-request.ts new file mode 100644 index 000000000..ea19d3aab --- /dev/null +++ b/src/submit-request.ts @@ -0,0 +1,415 @@ +import ot = require('./ot'); +import projections = require('./projections'); +import ShareDBError = require('./error'); +import types = require('./types'); +import protocol = require('./protocol'); + +var ERROR_CODE = ShareDBError.CODES; + +class SubmitRequest { + backend; + agent; + index; + projection; + collection; + id; + op; + options; + extra; + start; + action; + custom; + saveMilestoneSnapshot; + suppressPublish; + maxRetries; + retries; + snapshot; + ops; + channels; + _fixupOps; + + constructor(backend, agent, index, id, op, options) { + this.backend = backend; + this.agent = agent; + // If a projection, rewrite the call into a call against the collection + var projection = backend.projections[index]; + this.index = index; + this.projection = projection; + this.collection = (projection) ? projection.target : index; + this.id = id; + this.op = op; + this.options = options; + + this.extra = op.x; + delete op.x; + + this.start = Date.now(); + this._addOpMeta(); + + // Set as this request is sent through middleware + this.action = null; + // For custom use in middleware + this.custom = Object.create(null); + + // Whether or not to store a milestone snapshot. If left as null, the milestone + // snapshots are saved according to the interval provided to the milestone db + // options. If overridden to a boolean value, then that value is used instead of + // the interval logic. + this.saveMilestoneSnapshot = null; + this.suppressPublish = backend.suppressPublish; + this.maxRetries = backend.maxSubmitRetries; + this.retries = 0; + + // return values + this.snapshot = null; + this.ops = []; + this.channels = null; + this._fixupOps = []; + } + + $fixup(op) { + if (this.action !== this.backend.MIDDLEWARE_ACTIONS.apply) { + throw new ShareDBError( + ERROR_CODE.ERR_FIXUP_IS_ONLY_VALID_ON_APPLY, + 'fixup can only be called during the apply middleware' + ); + } + + if (this.op.del) { + throw new ShareDBError( + ERROR_CODE.ERR_CANNOT_FIXUP_DELETION, + 'fixup cannot be applied on deletion ops' + ); + } + + var typeId = this.op.create ? this.op.create.type : this.snapshot.type; + var type = types.map[typeId]; + if (typeof type.compose !== 'function') { + throw new ShareDBError( + ERROR_CODE.ERR_TYPE_DOES_NOT_SUPPORT_COMPOSE, + typeId + ' does not support compose' + ); + } + + if (this.op.create) this.op.create.data = type.apply(this.op.create.data, op); + else this.op.op = type.compose(this.op.op, op); + + var fixupOp = { + src: this.op.src, + seq: this.op.seq, + v: this.op.v, + op: op + }; + + this._fixupOps.push(fixupOp); + } + + submit(callback) { + var request = this; + var backend = this.backend; + var collection = this.collection; + var id = this.id; + var op = this.op; + // Send a special projection so that getSnapshot knows to return all fields. + // With a null projection, it strips document metadata + var fields = {$submit: true}; + + var snapshotOptions = {}; + snapshotOptions.agentCustom = request.agent.custom; + backend.db.getSnapshot(collection, id, fields, snapshotOptions, function(err, snapshot) { + if (err) return callback(err); + + request.snapshot = snapshot; + request._addSnapshotMeta(); + + if (op.v == null) { + if (op.create && snapshot.type && op.src) { + // If the document was already created by another op, we will return a + // 'Document already exists' error in response and fail to submit this + // op. However, this could also happen in the case that the op was + // already committed and the create op was simply resent. In that + // case, we should return a non-fatal 'Op already submitted' error. We + // must get the past ops and check their src and seq values to + // differentiate. + request._fetchCreateOpVersion(function(error, version) { + if (error) return callback(error); + if (version == null) { + callback(request.alreadyCreatedError()); + } else { + op.v = version; + callback(request.alreadySubmittedError()); + } + }); + return; + } + + // Submitting an op with a null version means that it should get the + // version from the latest snapshot. Generally this will mean the op + // won't be transformed, though transform could be called on it in the + // case of a retry from a simultaneous submit + op.v = snapshot.v; + } + + if (op.v === snapshot.v) { + // The snapshot hasn't changed since the op's base version. Apply + // without transforming the op + return request.apply(callback); + } + + if (op.v > snapshot.v) { + // The op version should be from a previous snapshot, so it should never + // never exceed the current snapshot's version + return callback(request.newerVersionError()); + } + + // Transform the op up to the current snapshot version, then apply + var from = op.v; + backend.db.getOpsToSnapshot(collection, id, from, snapshot, {metadata: true}, function(err, ops) { + if (err) return callback(err); + + if (ops.length !== snapshot.v - from) { + return callback(request.missingOpsError()); + } + + err = request._transformOp(ops); + if (err) return callback(err); + + var skipNoOp = backend.doNotCommitNoOps && + protocol.checkAtLeast(request.agent.protocol, '1.2') && + request.op.op && + request.op.op.length === 0; + + if (skipNoOp) { + // The op is a no-op, either because it was submitted as such, or - more + // likely - because it was transformed into one. Let's avoid committing it + // and tell the client. + return callback(request.noOpError()); + } + + if (op.v !== snapshot.v) { + // This shouldn't happen, but is just a final sanity check to make + // sure we have transformed the op to the current snapshot version + return callback(request.versionAfterTransformError()); + } + + request.apply(callback); + }); + }); + } + + apply(callback) { + // If we're being projected, verify that the op is allowed + var projection = this.projection; + if (projection && !projections.isOpAllowed(this.snapshot.type, projection.fields, this.op)) { + return callback(this.projectionError()); + } + + // Always set the channels before each attempt to apply. If the channels are + // modified in a middleware and we retry, we want to reset to a new array + this.channels = this.backend.getChannels(this.collection, this.id); + this._fixupOps = []; + delete this.op.m.fixup; + + var request = this; + this.backend.trigger(this.backend.MIDDLEWARE_ACTIONS.apply, this.agent, this, function(err) { + if (err) return callback(err); + + // Apply the submitted op to the snapshot + err = ot.apply(request.snapshot, request.op); + if (err) return callback(err); + + request.commit(callback); + }); + } + + commit(callback) { + var request = this; + var backend = this.backend; + backend.trigger(backend.MIDDLEWARE_ACTIONS.commit, this.agent, this, function(err) { + if (err) return callback(err); + if (request._fixupOps.length) request.op.m.fixup = request._fixupOps; + if (request.op.create) { + // When we create the snapshot, we store a pointer to the op that created + // it. This allows us to return OP_ALREADY_SUBMITTED errors when appropriate. + request.snapshot.m._create = { + src: request.op.src, + seq: request.op.seq, + v: request.op.v + }; + } + + // Try committing the operation and snapshot to the database atomically + backend.db.commit( + request.collection, + request.id, + request.op, + request.snapshot, + request.options, + function(err, succeeded) { + if (err) return callback(err); + if (!succeeded) { + // Between our fetch and our call to commit, another client committed an + // operation. We expect this to be relatively infrequent but normal. + return request.retry(callback); + } + if (!request.suppressPublish) { + var op = request.op; + op.c = request.collection; + op.d = request.id; + op.m = undefined; + // Needed for agent to detect if it can ignore sending the op back to + // the client that submitted it in subscriptions + if (request.collection !== request.index) op.i = request.index; + backend.pubsub.publish(request.channels, op); + } + if (request._shouldSaveMilestoneSnapshot(request.snapshot)) { + request.backend.milestoneDb.saveMilestoneSnapshot(request.collection, request.snapshot); + } + callback(); + }); + }); + } + + retry(callback) { + this.retries++; + if (this.maxRetries != null && this.retries > this.maxRetries) { + return callback(this.maxRetriesError()); + } + this.backend.emit('timing', 'submit.retry', Date.now() - this.start, this); + this.submit(callback); + } + + _transformOp(ops) { + var type = this.snapshot.type; + for (var i = 0; i < ops.length; i++) { + var op = ops[i]; + + if (this.op.src && this.op.src === op.src && this.op.seq === op.seq) { + // The op has already been submitted. There are a variety of ways this + // can happen in normal operation, such as a client resending an + // unacknowledged operation at reconnect. It's important we don't apply + // the same op twice + if (op.m.fixup) this._fixupOps = op.m.fixup; + return this.alreadySubmittedError(); + } + + if (this.op.v !== op.v) { + return this.versionDuringTransformError(); + } + + var err = ot.transform(type, this.op, op); + if (err) return err; + delete op.m; + this.ops.push(op); + } + } + + _addOpMeta() { + this.op.m = { + ts: this.start + }; + if (this.op.create) { + // Consistently store the full URI of the type, not just its short name + this.op.create.type = ot.normalizeType(this.op.create.type); + } + } + + _addSnapshotMeta() { + var meta = this.snapshot.m || (this.snapshot.m = {}); + if (this.op.create) { + meta.ctime = this.start; + } else if (this.op.del) { + this.op.m.data = this.snapshot.data; + } + meta.mtime = this.start; + } + + _shouldSaveMilestoneSnapshot(snapshot) { + // If the flag is null, it's not been overridden by the consumer, so apply the interval + if (this.saveMilestoneSnapshot === null) { + return snapshot && snapshot.v % this.backend.milestoneDb.interval === 0; + } + + return this.saveMilestoneSnapshot; + } + + _fetchCreateOpVersion(callback) { + var create = this.snapshot.m._create; + if (create) { + var version = (create.src === this.op.src && create.seq === this.op.seq) ? create.v : null; + return callback(null, version); + } + + // We can only reach here if the snapshot is missing the create metadata. + // This can happen if a client tries to re-create or resubmit a create op to + // a "legacy" snapshot that existed before we started adding the meta (should + // be uncommon) or when using a driver that doesn't support metadata (eg Postgres). + this.backend.db.getCommittedOpVersion(this.collection, this.id, this.snapshot, this.op, null, callback); + } + + // Non-fatal client errors: + alreadySubmittedError() { + return new ShareDBError(ERROR_CODE.ERR_OP_ALREADY_SUBMITTED, 'Op already submitted'); + } + + rejectedError() { + return new ShareDBError(ERROR_CODE.ERR_OP_SUBMIT_REJECTED, 'Op submit rejected'); + } + + // Fatal client errors: + alreadyCreatedError() { + return new ShareDBError(ERROR_CODE.ERR_DOC_ALREADY_CREATED, 'Invalid op submitted. Document already created'); + } + + newerVersionError() { + return new ShareDBError( + ERROR_CODE.ERR_OP_VERSION_NEWER_THAN_CURRENT_SNAPSHOT, + 'Invalid op submitted. Op version newer than current snapshot' + ); + } + + projectionError() { + return new ShareDBError( + ERROR_CODE.ERR_OP_NOT_ALLOWED_IN_PROJECTION, + 'Invalid op submitted. Operation invalid in projected collection' + ); + } + + // Fatal internal errors: + missingOpsError() { + return new ShareDBError( + ERROR_CODE.ERR_SUBMIT_TRANSFORM_OPS_NOT_FOUND, + 'Op submit failed. DB missing ops needed to transform it up to the current snapshot version' + ); + } + + versionDuringTransformError() { + return new ShareDBError( + ERROR_CODE.ERR_OP_VERSION_MISMATCH_DURING_TRANSFORM, + 'Op submit failed. Versions mismatched during op transform' + ); + } + + versionAfterTransformError() { + return new ShareDBError( + ERROR_CODE.ERR_OP_VERSION_MISMATCH_AFTER_TRANSFORM, + 'Op submit failed. Op version mismatches snapshot after op transform' + ); + } + + maxRetriesError() { + return new ShareDBError( + ERROR_CODE.ERR_MAX_SUBMIT_RETRIES_EXCEEDED, + 'Op submit failed. Exceeded max submit retries of ' + this.maxRetries + ); + } + + noOpError() { + return new ShareDBError( + ERROR_CODE.ERR_NO_OP, + 'Op is a no-op. Skipping apply.' + ); + } +} + +export = SubmitRequest; diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 000000000..28df18125 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,10 @@ + +export const defaultType = require('ot-json0').type; +export const map = Object.create(null); + +export function register(type) { + if (type.name) exports.map[type.name] = type; + if (type.uri) exports.map[type.uri] = type; +} + +exports.register(exports.defaultType); diff --git a/lib/util.js b/src/util.ts similarity index 70% rename from lib/util.js rename to src/util.ts index 922350492..d31b4a85e 100644 --- a/lib/util.js +++ b/src/util.ts @@ -1,123 +1,123 @@ -var nextTickImpl = require('./next-tick'); - -exports.doNothing = doNothing; -function doNothing() {} - -exports.hasKeys = function(object) { - for (var key in object) return true; - return false; -}; - -var hasOwn; -exports.hasOwn = hasOwn = Object.hasOwn || function(obj, prop) { - return Object.prototype.hasOwnProperty.call(obj, prop); -}; - -// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/isInteger#Polyfill -exports.isInteger = Number.isInteger || function(value) { - return typeof value === 'number' && - isFinite(value) && - Math.floor(value) === value; -}; - -exports.isValidVersion = function(version) { - if (version === null) return true; - return exports.isInteger(version) && version >= 0; -}; - -exports.isValidTimestamp = function(timestamp) { - return exports.isValidVersion(timestamp); -}; - -exports.MAX_SAFE_INTEGER = 9007199254740991; - -exports.dig = function() { - var obj = arguments[0]; - for (var i = 1; i < arguments.length; i++) { - var key = arguments[i]; - obj = hasOwn(obj, key) ? obj[key] : (i === arguments.length - 1 ? undefined : Object.create(null)); - } - return obj; -}; - -exports.digOrCreate = function() { - var obj = arguments[0]; - var createCallback = arguments[arguments.length - 1]; - for (var i = 1; i < arguments.length - 1; i++) { - var key = arguments[i]; - obj = hasOwn(obj, key) ? obj[key] : - (obj[key] = i === arguments.length - 2 ? createCallback() : Object.create(null)); - } - return obj; -}; - -exports.digAndRemove = function() { - var obj = arguments[0]; - var objects = [obj]; - for (var i = 1; i < arguments.length - 1; i++) { - var key = arguments[i]; - if (!hasOwn(obj, key)) break; - obj = obj[key]; - objects.push(obj); - }; - - for (var i = objects.length - 1; i >= 0; i--) { - var parent = objects[i]; - var key = arguments[i + 1]; - var child = parent[key]; - if (i === objects.length - 1 || !exports.hasKeys(child)) delete parent[key]; - } -}; - -exports.supportsPresence = function(type) { - return type && typeof type.transformPresence === 'function'; -}; - -exports.callEach = function(callbacks, error) { - var called = false; - callbacks.forEach(function(callback) { - if (callback) { - callback(error); - called = true; - } - }); - return called; -}; - -exports.truthy = function(arg) { - return !!arg; -}; - -if (typeof process !== 'undefined' && typeof process.nextTick === 'function') { - exports.nextTick = process.nextTick; -} else if (typeof MessageChannel !== 'undefined') { - exports.nextTick = nextTickImpl.messageChannel; -} else { - exports.nextTick = nextTickImpl.setTimeout; -} - -exports.clone = function(obj) { - return (obj === undefined) ? undefined : JSON.parse(JSON.stringify(obj)); -}; - -var objectProtoPropNames = Object.create(null); -Object.getOwnPropertyNames(Object.prototype).forEach(function(prop) { - if (prop !== '__proto__') { - objectProtoPropNames[prop] = true; - } -}); -exports.isDangerousProperty = function(propName) { - return propName === '__proto__' || objectProtoPropNames[propName]; -}; - -try { - var util = require('util'); - if (typeof util.inherits !== 'function') throw new Error('Could not find util.inherits()'); - exports.inherits = util.inherits; -} catch (e) { - try { - exports.inherits = require('inherits'); - } catch (e) { - throw new Error('If running sharedb in a browser, please install the "inherits" or "util" package'); - } -} +import nextTickImpl = require('./next-tick'); +export let nextTick: any; +export let inherits: any; +export function doNothing() {} + +export function hasKeys(object) { + for (var key in object) return true; + return false; +} + +export const hasOwn = Object.hasOwn || function(obj, prop) { + return Object.prototype.hasOwnProperty.call(obj, prop); +}; + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/isInteger#Polyfill +export const isInteger = Number.isInteger || function(value) { + return typeof value === 'number' && + isFinite(value) && + Math.floor(value) === value; +}; + +export function isValidVersion(version) { + if (version === null) return true; + return exports.isInteger(version) && version >= 0; +} + +export function isValidTimestamp(timestamp) { + return exports.isValidVersion(timestamp); +} + +export const MAX_SAFE_INTEGER = 9007199254740991; + +export function dig() { + var obj = arguments[0]; + for (var i = 1; i < arguments.length; i++) { + var key = arguments[i]; + obj = hasOwn(obj, key) ? obj[key] : (i === arguments.length - 1 ? undefined : Object.create(null)); + } + return obj; +} + +export function digOrCreate() { + var obj = arguments[0]; + var createCallback = arguments[arguments.length - 1]; + for (var i = 1; i < arguments.length - 1; i++) { + var key = arguments[i]; + obj = hasOwn(obj, key) ? obj[key] : + (obj[key] = i === arguments.length - 2 ? createCallback() : Object.create(null)); + } + return obj; +} + +export function digAndRemove() { + var obj = arguments[0]; + var objects = [obj]; + for (var i = 1; i < arguments.length - 1; i++) { + var key = arguments[i]; + if (!hasOwn(obj, key)) break; + obj = obj[key]; + objects.push(obj); + }; + + for (var i = objects.length - 1; i >= 0; i--) { + var parent = objects[i]; + var key = arguments[i + 1]; + var child = parent[key]; + if (i === objects.length - 1 || !exports.hasKeys(child)) delete parent[key]; + } +} + +export function supportsPresence(type) { + return type && typeof type.transformPresence === 'function'; +} + +export function callEach(callbacks, error) { + var called = false; + callbacks.forEach(function(callback) { + if (callback) { + callback(error); + called = true; + } + }); + return called; +} + +export function truthy(arg) { + return !!arg; +} + +if (typeof process !== 'undefined' && typeof process.nextTick === 'function') { + nextTick = process.nextTick; +} else if (typeof MessageChannel !== 'undefined') { + nextTick = nextTickImpl.messageChannel; +} else { + nextTick = nextTickImpl.setTimeout; +} + +export function clone(obj) { + return (obj === undefined) ? undefined : JSON.parse(JSON.stringify(obj)); +} + +var objectProtoPropNames = Object.create(null); +Object.getOwnPropertyNames(Object.prototype).forEach(function(prop) { + if (prop !== '__proto__') { + objectProtoPropNames[prop] = true; + } +}); + +export function isDangerousProperty(propName) { + return propName === '__proto__' || objectProtoPropNames[propName]; +} + +try { + var util = require('util'); + if (typeof util.inherits !== 'function') throw new Error('Could not find util.inherits()'); + inherits = util.inherits; +} catch (e) { + try { + inherits = require('inherits'); + } catch (e) { + throw new Error('If running sharedb in a browser, please install the "inherits" or "util" package'); + } +} diff --git a/test/client/query-subscribe.js b/test/client/query-subscribe.js index 4cf690ff9..673299528 100644 --- a/test/client/query-subscribe.js +++ b/test/client/query-subscribe.js @@ -13,6 +13,9 @@ module.exports = function(options) { }); afterEach(function() { + if (sinon.clock) { + sinon.clock.uninstall(); + } sinon.restore(); }); @@ -517,7 +520,7 @@ function commonTests(options) { }); it('pollDebounce option reduces subsequent poll interval', function(done) { - var clock = sinon.useFakeTimers(); + var clock = util.useFakeTimers(); var connection = this.backend.connect(); this.backend.db.canPollDoc = function() { return false; @@ -566,7 +569,7 @@ function commonTests(options) { }); it('db.pollDebounce option reduces subsequent poll interval', function(done) { - var clock = sinon.useFakeTimers(); + var clock = util.useFakeTimers(); var connection = this.backend.connect(); this.backend.db.canPollDoc = function() { return false; @@ -616,7 +619,7 @@ function commonTests(options) { }); it('pollInterval updates a subscribed query after an unpublished create', function(done) { - var clock = sinon.useFakeTimers(); + var clock = util.useFakeTimers(); var connection = this.backend.connect(); this.backend.suppressPublish = true; var query = connection.createSubscribeQuery( @@ -638,7 +641,7 @@ function commonTests(options) { }); it('db.pollInterval updates a subscribed query after an unpublished create', function(done) { - var clock = sinon.useFakeTimers(); + var clock = util.useFakeTimers(); var connection = this.backend.connect(); this.backend.suppressPublish = true; this.backend.db.pollDebounce = 0; @@ -657,7 +660,7 @@ function commonTests(options) { }); it('pollInterval captures additional unpublished creates', function(done) { - var clock = sinon.useFakeTimers(); + var clock = util.useFakeTimers(); var connection = this.backend.connect(); this.backend.suppressPublish = true; var count = 0; diff --git a/test/client/snapshot-timestamp-request.js b/test/client/snapshot-timestamp-request.js index 5ba12ee8d..1beaaf4fd 100644 --- a/test/client/snapshot-timestamp-request.js +++ b/test/client/snapshot-timestamp-request.js @@ -2,6 +2,7 @@ var Backend = require('../../lib/backend'); var expect = require('chai').expect; var MemoryDb = require('../../lib/db/memory'); var MemoryMilestoneDb = require('../../lib/milestone-db/memory'); +var util = require('../util'); var sinon = require('sinon'); var async = require('async'); @@ -17,12 +18,12 @@ describe('SnapshotTimestampRequest', function() { var ONE_DAY = 1000 * 60 * 60 * 24; beforeEach(function() { - clock = sinon.useFakeTimers(day1); + clock = util.useFakeTimers(day1); backend = new Backend(); }); afterEach(function(done) { - clock.uninstall(); + sinon.restore(); backend.close(done); }); diff --git a/test/setup.js b/test/setup.js index d00aeee38..4d3aca185 100644 --- a/test/setup.js +++ b/test/setup.js @@ -3,7 +3,7 @@ var sinon = require('sinon'); var sinonChai = require('sinon-chai'); var chai = require('chai'); -chai.use(sinonChai); +chai.use(sinonChai.default); if (process.env.LOGGING !== 'true') { // Silence the logger for tests by setting all its methods to no-ops @@ -15,5 +15,8 @@ if (process.env.LOGGING !== 'true') { } afterEach(function() { + if (sinon.clock) { + sinon.clock.uninstall(); + } sinon.restore(); }); diff --git a/test/util.js b/test/util.js index 17fa6500f..812fb7571 100644 --- a/test/util.js +++ b/test/util.js @@ -1,3 +1,4 @@ +var sinon = require('sinon'); exports.sortById = function(docs) { return docs.slice().sort(function(a, b) { @@ -15,6 +16,17 @@ exports.pluck = function(docs, key) { return values; }; +/** + * @param {Parameters[0]} config + * @see {@link sinon.useFakeTimers} + * @see {@link https://github.com/sinonjs/fake-timers#clocksettickmodemode} + */ +exports.useFakeTimers = function(config) { + var clock = sinon.useFakeTimers(config); + clock.setTickMode({mode: 'nextAsync'}); + return clock; +}; + // Wrap a done function to call back only after a specified number of calls. // For example, `var callbackAfter = callAfter(1, callback)` means that if // `callbackAfter` is called once, it won't call back. If it is called twice diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 000000000..c6ce45f1b --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,29 @@ +{ + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "target": "ES5", + "module": "CommonJS", + "types": [ + "node" + ], + "strict": true, + "esModuleInterop": false, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "ignoreDeprecations": "6.0", + "noImplicitAny": false, + "noImplicitThis": false, + "removeComments": false, + // Disable specific strict checks until the codebase is more properly typed + "strictNullChecks": false, + "useUnknownInCatchVariables": false, + "alwaysStrict": false, + "strictFunctionTypes": false, + "strictBindCallApply": false, + "strictPropertyInitialization": false, + }, + "include": [ + "src/**/*.ts" + ], +} \ No newline at end of file