From 45611b92938864a70dddda54b3c36b824095cb90 Mon Sep 17 00:00:00 2001 From: Chrystian Huot Date: Tue, 26 Nov 2019 09:08:50 -0500 Subject: [PATCH] New version 2.1 --- README.md | 6 +- client/package.json | 4 +- .../rdio-scanner-calls-query.service.ts | 23 +- .../rdio-scanner/rdio-scanner.component.html | 2 +- .../rdio-scanner/rdio-scanner.component.ts | 223 ++++++------------ package.json | 2 +- .../graphql/datasources/rdio-scanner-call.js | 6 +- server/graphql/queries/rdio-scanner-calls.gql | 5 +- server/graphql/queries/rdio-scanner-calls.js | 4 +- server/helpers/paginator.js | 31 --- ...191028144433-create-rdio-scanner-system.js | 2 +- ...20191029092201-create-rdio-scanner-call.js | 2 +- ...91126135515-optimize-rdio-scanner-calls.js | 22 ++ server/package.json | 12 +- 14 files changed, 124 insertions(+), 220 deletions(-) delete mode 100644 server/helpers/paginator.js create mode 100644 server/migrations/20191126135515-optimize-rdio-scanner-calls.js diff --git a/README.md b/README.md index a060691..3d99cde 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Rdio Scanner v2.0 +# Rdio Scanner v2.1 *Rdio Scanner* is a progressive web interface designed to resemble an old school radio scanner. It integrates all frontend / backend components to manage audio files from different sources. @@ -8,7 +8,9 @@ For now, only [Trunk Recorder](https://github.com/robotastic/trunk-recorder) sof ## What's new in this version -The version 2.0 of *Rdio Scanner* is a major rewrite in which Meteor has been replaced by [The Apollo Data Graph Platform](https://www.apollographql.com/) for its API. MongoDB is also replaced by [SQLite](https://www.sqlite.org/) to facilitate the entire installation process. It is still possible to use another database by changing [Sequelize ORM](https://sequelize.org/) settings accordingly. +Version 2.1 focuses on various speed improvements for searching stored calls. + +Version 2.0 is a major rewrite in which Meteor has been replaced by [The Apollo Data Graph Platform](https://www.apollographql.com/) for its API. MongoDB is also replaced by [SQLite](https://www.sqlite.org/) to facilitate the entire installation process. It is still possible to use another database by changing [Sequelize ORM](https://sequelize.org/) settings accordingly. These changes bring many *performance benefits* to *Rdio scanner* and make it *much easier to install*. diff --git a/client/package.json b/client/package.json index 07d78c7..9d6348a 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "rdio-scanner-client", - "version": "2.0.0", + "version": "2.1.0", "private": true, "scripts": { "build": "ng build --cross-origin use-credentials --prod $*", @@ -38,7 +38,7 @@ "@angular/cli": "~8.3.19", "@angular/compiler-cli": "~8.2.14", "@angular/language-service": "~8.2.14", - "@types/node": "~12.12.8", + "@types/node": "~12.12.14", "codelyzer": "^5.2.0", "ts-node": "~8.5.2", "tslint": "~5.20.1", diff --git a/client/src/app/components/rdio-scanner/rdio-scanner-calls-query.service.ts b/client/src/app/components/rdio-scanner/rdio-scanner-calls-query.service.ts index 54e517a..9cca5f6 100644 --- a/client/src/app/components/rdio-scanner/rdio-scanner-calls-query.service.ts +++ b/client/src/app/components/rdio-scanner/rdio-scanner-calls-query.service.ts @@ -46,18 +46,16 @@ export class AppRdioScannerCallsQueryService extends Query + [pageSize]="searchResults.value.length" [showFirstLastButtons]="true">
diff --git a/client/src/app/components/rdio-scanner/rdio-scanner.component.ts b/client/src/app/components/rdio-scanner/rdio-scanner.component.ts index ec9a996..4dd1863 100644 --- a/client/src/app/components/rdio-scanner/rdio-scanner.component.ts +++ b/client/src/app/components/rdio-scanner/rdio-scanner.component.ts @@ -1,8 +1,7 @@ -import { AfterViewInit, Component, HostListener, NgZone, OnDestroy, OnInit, ViewChild } from '@angular/core'; +import { Component, HostListener, NgZone, OnDestroy, OnInit, ViewChild } from '@angular/core'; import { FormBuilder, FormGroup } from '@angular/forms'; -import { MatPaginator, PageEvent } from '@angular/material/paginator'; -import { MatTable, MatTableDataSource } from '@angular/material/table'; -import { Subscription } from 'rxjs'; +import { MatPaginator } from '@angular/material/paginator'; +import { BehaviorSubject, Subscription } from 'rxjs'; import { AppRdioScannerCallQueryService } from './rdio-scanner-call-query.service'; import { AppRdioScannerCallSubscriptionService, RdioScannerCall } from './rdio-scanner-call-subscription.service'; import { AppRdioScannerCallsQueryService } from './rdio-scanner-calls-query.service'; @@ -24,7 +23,7 @@ interface RdioScannerSelection { styleUrls: ['./rdio-scanner.component.scss'], templateUrl: './rdio-scanner.component.html', }) -export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit { +export class AppRdioScannerComponent implements OnDestroy, OnInit { get call() { return this._call; } get callHistory() { return this._callHistory; } get callPrevious() { return this._callPrevious; } @@ -82,7 +81,7 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit private _searchFormDateStart = ''; private _searchFormDateStop = ''; private _searchPanelOpened = false; - private _searchResults = new MatTableDataSource(); + private _searchResults = new BehaviorSubject(new Array(10)); private _searchResultsCount = 0; private _searchResultsPending = false; private _selection: RdioScannerSelection = {}; @@ -98,10 +97,12 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit private livefeedSubscription: Subscription; private searchFormSubscription: Subscription; private searchPaginatorSubscription: Subscription; + private searchResultsBuffer = new Array(200); + private searchResultsBufferFrom = 0; + private searchResultsBufferTo = this.searchResultsBuffer.length - 1; private systemsSubscription: Subscription; @ViewChild(MatPaginator, { static: true }) private matPaginator: MatPaginator; - @ViewChild(MatTable, { static: true }) private matTable: MatTable; constructor( private appRdioScannerCallQuery: AppRdioScannerCallQueryService, @@ -282,19 +283,13 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit loadAndPlay(call: RdioScannerCall): void { this.appRdioScannerCallQuery.fetch({ id: call.id }).subscribe(({ data }) => { if (data.rdioScannerCall) { - Object.assign(call, data.rdioScannerCall); + Object.assign(call, this.transformCall(data.rdioScannerCall)); this.play(call); } }); } - ngAfterViewInit(): void { - if (this.matPaginator instanceof MatPaginator) { - this.searchResults.paginator = this.matPaginator; - } - } - ngOnDestroy(): void { this.unsubscribeLivefeed(); @@ -314,7 +309,6 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit this.subscribeSystems().then(() => { this.subscribeSearchForm(); - this.subscribeSearchPaginator(); }); } @@ -430,7 +424,7 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit toggleSearchPanel(opened = !this.searchPanelOpened): void { if (opened) { - this.searchCalls(); + this.loadStoredCalls(); } this._searchPanelOpened = opened; @@ -525,97 +519,70 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit }, this.selection || {}); } - private searchCalls(paginator: { - first?: boolean | number; - last?: boolean | number; - skip?: number; - } = {}): void { - const options: { - date?: Date; - first?: number; - last?: number; - skip?: number; - sort?: number; - system?: number; - talkgroup?: number; - } = {}; + private async loadStoredCalls(filters?: any): Promise { + const limit = this.searchResultsBuffer.length; - const count = 200; + const pageFrom = this.matPaginator.pageIndex * this.matPaginator.pageSize; - const form = this.searchForm.value; + const pageTo = this.matPaginator.pageIndex * this.matPaginator.pageSize + this.matPaginator.pageSize - 1; - if (form.date instanceof Date) { - options.date = form.date; - } + if (!this.searchResultsPending && (filters || !pageFrom || + pageFrom < this.searchResultsBufferFrom || pageTo > this.searchResultsBufferTo)) { - if (typeof paginator.first === 'boolean') { - options.first = count; + filters = filters || this.searchForm.value; - } else if (typeof paginator.first === 'number') { - options.first = paginator.first; + this.searchResultsBufferFrom = Math.floor(pageFrom / limit) * limit; + this.searchResultsBufferTo = this.searchResultsBufferFrom + limit - 1; - } else if (typeof paginator.last === 'boolean') { - options.last = count; + const options: { + date?: Date; + limit?: number; + offset?: number; + sort?: number; + system?: number; + talkgroup?: number; + } = {}; - } else if (typeof paginator.last === 'number') { - options.last = paginator.last; - - } else { - options.first = count; - } - - if (typeof paginator.skip === 'number' && paginator.skip > 0) { - options.skip = paginator.skip; - } - - if (typeof form.sort === 'number') { - options.sort = form.sort; - } - - if (typeof form.system === 'number' && form.system !== -1) { - options.system = form.system; - } - - if (typeof form.talkgroup === 'number' && form.talkgroup !== -1) { - options.talkgroup = form.talkgroup; - } - - this._searchResultsPending = true; - - this.searchForm.disable(); - - this.appRdioScannerCallsQuery.fetch(options, { fetchPolicy: 'no-cache' }).subscribe(({ data }) => { - const results = data.rdioScannerCalls.results.map((call) => this.transformCall(call)); - - if (this.searchResults.data.length < data.rdioScannerCalls.count) { - const ar = new Array(data.rdioScannerCalls.count - this.searchResults.data.length); - - if (options.sort < 0) { - this.searchResults.data.unshift(...ar); - } else if (options.sort > 0) { - this.searchResults.data.push(...ar); - } + if (filters.date instanceof Date) { + options.date = filters.date; } - if (options.last) { - this.searchResults.data.splice((options.skip || this.searchResults.data.length) - results.length, - results.length, ...results); - - } else { - this.searchResults.data.splice(options.skip || 0, results.length, ...results); + if (limit) { + options.limit = limit; } - this._searchResultsCount = data.rdioScannerCalls.count; - this._searchFormDateStart = data.rdioScannerCalls.dateStart; - this._searchFormDateStop = data.rdioScannerCalls.dateStop; + if (this.searchResultsBufferFrom) { + options.offset = this.searchResultsBufferFrom; + } - this.matTable.renderRows(); - this.searchResults.data = this.searchResults.data.slice(); // because renderRow() doesn't want to work + if (filters.sort < 0) { + options.sort = -1; + } - this.searchForm.enable(); + if (filters.system >= 0) { + options.system = filters.system; + } + + if (filters.talkgroup >= 0) { + options.talkgroup = filters.talkgroup; + } + + this._searchResultsPending = true; + + const query = await this.appRdioScannerCallsQuery.fetch(options, { fetchPolicy: 'no-cache' }).toPromise(); + + for (let i = 0; i < limit; i++) { + this.searchResultsBuffer[i] = this.transformCall(query.data.rdioScannerCalls.results[i]); + } + + this._searchResultsCount = query.data.rdioScannerCalls.count; + this._searchFormDateStart = query.data.rdioScannerCalls.dateStart; + this._searchFormDateStop = query.data.rdioScannerCalls.dateStop; this._searchResultsPending = false; - }); + } + + this.searchResults.next(this.searchResultsBuffer.slice(pageFrom % limit, pageTo % limit + 1)); } private async subscribeLivefeed(): Promise { @@ -632,23 +599,17 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit if (!this.searchFormSubscription) { this.searchFormSubscription = this.searchForm.valueChanges.subscribe((value) => { - if (!this.searchResultsPending) { - if (!Object.keys(value).every((key) => value[key] === lastValue[key])) { - if (value.system !== lastValue.system) { - this.searchForm.get('talkgroup').setValue(-1, { emitEvent: false }); - } + if (!Object.keys(value).every((key) => value[key] === lastValue[key])) { + lastValue = value; - lastValue = this.searchForm.value; - - this.searchResults.data.splice(0, this.searchResults.data.length, - ...new Array(this.matPaginator.pageSize)); - - this.searchCalls(); - - if (this.matPaginator.pageIndex > 0) { - this.matPaginator.firstPage(); - } + if (value.system !== lastValue.system && value.talkgroup !== -1) { + lastValue.talkgroup = -1; + this.searchForm.get('talkgroup').reset(lastValue.talkgroup); } + + this.matPaginator.firstPage(); + + this.loadStoredCalls(value); } }); } @@ -656,43 +617,7 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit private subscribeSearchPaginator(): void { if (!this.searchPaginatorSubscription) { - this.searchPaginatorSubscription = this.matPaginator.page.subscribe((event: PageEvent) => { - const sortOrder = this.searchForm.get('sort').value; - - if (sortOrder < 0 && event.pageIndex === 0) { - this.searchCalls({ first: true }); - - } else if (sortOrder > 0 && event.pageIndex >= event.length / event.pageSize - 1) { - this.searchCalls({ last: true }); - - } else if (event.length - event.pageIndex * event.pageSize < event.pageSize) { - const needMore = !this.searchResults.data - .slice(event.pageIndex * event.pageSize, event.pageIndex * event.pageSize + event.pageSize) - .every((call) => call); - - if (needMore) { - this.searchCalls({ last: true, skip: event.length }); - } - - } else if (event.pageIndex > event.previousPageIndex) { - const needMore = !this.searchResults.data - .slice(event.pageIndex * event.pageSize, event.pageIndex * event.pageSize + event.pageSize) - .every((call) => call); - - if (needMore) { - this.searchCalls({ first: true, skip: event.pageIndex * event.pageSize }); - } - - } else if (event.pageIndex < event.previousPageIndex) { - const needMore = !this.searchResults.data - .slice(event.pageIndex * event.pageSize - 1, event.pageIndex * event.pageSize + event.pageSize - 1) - .every((call) => call); - - if (needMore) { - this.searchCalls({ last: true, skip: event.pageIndex * event.pageSize + event.pageSize - 1 }); - } - } - }); + this.searchPaginatorSubscription = this.matPaginator.page.subscribe(() => this.loadStoredCalls()); } } @@ -707,11 +632,13 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit } private transformCall(call: RdioScannerCall): RdioScannerCall { - call.systemData = (call && this.systems - .find((system: RdioScannerSystem) => system.system === call.system)) || {}; + if (call) { + call.systemData = (call && this.systems + .find((system: RdioScannerSystem) => system.system === call.system)) || {}; - call.talkgroupData = (call && call.systemData && Array.isArray(call.systemData.talkgroups) && call.systemData.talkgroups - .find((talkgroup: RdioScannerTalkgroup) => talkgroup.dec === call.talkgroup)) || {}; + call.talkgroupData = (call && call.systemData && Array.isArray(call.systemData.talkgroups) && call.systemData.talkgroups + .find((talkgroup: RdioScannerTalkgroup) => talkgroup.dec === call.talkgroup)) || {}; + } return call; } diff --git a/package.json b/package.json index 4abe57b..8d968c2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rdio-scanner", - "version": "2.0.0", + "version": "2.1.0", "private": true, "main": "run.js", "scripts": { diff --git a/server/graphql/datasources/rdio-scanner-call.js b/server/graphql/datasources/rdio-scanner-call.js index 45c07e2..bdd028e 100644 --- a/server/graphql/datasources/rdio-scanner-call.js +++ b/server/graphql/datasources/rdio-scanner-call.js @@ -1,7 +1,6 @@ 'use strict'; const { DataSource } = require('apollo-datasource'); -const { getPageRange } = require('../../helpers/paginator'); const { callReducer } = require('../../helpers/rdio-scanner'); class RdioScannerSystem extends DataSource { @@ -21,7 +20,7 @@ class RdioScannerSystem extends DataSource { return callReducer(result.dataValues); } - async getCalls({ date, system, talkgroup }, { first, last, skip, sort }) { + async getCalls({ date, system, talkgroup }, { limit, offset, sort }) { const Op = this.store.Sequelize.Op; const attributes = { @@ -59,7 +58,8 @@ class RdioScannerSystem extends DataSource { const count = await this.store.rdioScannerCall.count({ where }); - const { limit, offset } = getPageRange({ count, first, last, skip }); + limit = typeof limit === 'number' ? limit : 100; + offset = typeof offset === 'number' ? offset : 0; const calls = await this.store.rdioScannerCall.findAll({ attributes, limit, offset, order, where }); diff --git a/server/graphql/queries/rdio-scanner-calls.gql b/server/graphql/queries/rdio-scanner-calls.gql index e26c71c..c613f18 100644 --- a/server/graphql/queries/rdio-scanner-calls.gql +++ b/server/graphql/queries/rdio-scanner-calls.gql @@ -1,8 +1,7 @@ rdioScannerCalls( date: Date - first: Int - last: Int - skip: Int + limit: Int + offset: Int sort: Int system: Int talkgroup: Int diff --git a/server/graphql/queries/rdio-scanner-calls.js b/server/graphql/queries/rdio-scanner-calls.js index 82b7045..2bb9531 100644 --- a/server/graphql/queries/rdio-scanner-calls.js +++ b/server/graphql/queries/rdio-scanner-calls.js @@ -1,7 +1,7 @@ 'use strict'; -module.exports = () => async (_, { date, first, last, skip, sort, system, talkgroup }, { dataSources }) => { - const { count, dateStart, dateStop, results } = await dataSources.rdioScannerCall.getCalls({ date, system, talkgroup }, { first, last, skip, sort }); +module.exports = () => async (_, { date, limit, offset, sort, system, talkgroup }, { dataSources }) => { + const { count, dateStart, dateStop, results } = await dataSources.rdioScannerCall.getCalls({ date, system, talkgroup }, { limit, offset, sort }); return { count, dateStart, dateStop, results }; }; diff --git a/server/helpers/paginator.js b/server/helpers/paginator.js deleted file mode 100644 index d27e2e8..0000000 --- a/server/helpers/paginator.js +++ /dev/null @@ -1,31 +0,0 @@ -'use strict'; - -function getPageRange({ count, first, last, skip }) { - const page = { - default: 100, - max: 1000, - }; - - let limit; - - let offset; - - first = typeof first === 'number' && first > 0 ? Math.min(page.max, first) : typeof last !== 'number' ? page.default : null; - - last = typeof last === 'number' && last > 0 ? Math.min(page.max, last) : null; - - skip = typeof skip === 'number' && skip > 0 ? skip : 0; - - if (typeof first === 'number') { - limit = Math.max(1, Math.min(count, skip + first) - skip); - offset = Math.min(count - 1, skip); - - } else { - limit = count - Math.max(0, Math.min(count, skip) - last); - offset = Math.max(0, Math.max(count, skip) - last); - } - - return { limit, offset }; -} - -module.exports = { getPageRange }; diff --git a/server/migrations/20191028144433-create-rdio-scanner-system.js b/server/migrations/20191028144433-create-rdio-scanner-system.js index 276d4bb..ca74cc0 100644 --- a/server/migrations/20191028144433-create-rdio-scanner-system.js +++ b/server/migrations/20191028144433-create-rdio-scanner-system.js @@ -59,5 +59,5 @@ module.exports = { throw err; } - } + }, }; diff --git a/server/migrations/20191029092201-create-rdio-scanner-call.js b/server/migrations/20191029092201-create-rdio-scanner-call.js index 3b509c6..1ae271c 100644 --- a/server/migrations/20191029092201-create-rdio-scanner-call.js +++ b/server/migrations/20191029092201-create-rdio-scanner-call.js @@ -87,5 +87,5 @@ module.exports = { throw err; } - } + }, }; diff --git a/server/migrations/20191126135515-optimize-rdio-scanner-calls.js b/server/migrations/20191126135515-optimize-rdio-scanner-calls.js new file mode 100644 index 0000000..a673384 --- /dev/null +++ b/server/migrations/20191126135515-optimize-rdio-scanner-calls.js @@ -0,0 +1,22 @@ +'use strict'; + +module.exports = { + + up: async (queryInterface, Sequelize) => { + const transaction = await queryInterface.sequelize.transaction(); + + try { + await queryInterface.removeIndex('rdioScannerCalls', ['system'], { transaction }); + + await queryInterface.removeIndex('rdioScannerCalls', ['talkgroup'], { transaction }); + + await transaction.commit(); + + } catch (err) { + await transaction.rollback(); + + throw err; + } + }, +}; + diff --git a/server/package.json b/server/package.json index ab42299..3250c4c 100644 --- a/server/package.json +++ b/server/package.json @@ -1,6 +1,6 @@ { "name": "rdio-scanner-server", - "version": "2.0.0", + "version": "2.1.0", "private": true, "main": "index.js", "scripts": { @@ -11,7 +11,7 @@ "license": "LICENSE", "dependencies": { "apollo-datasource": "^0.6.3", - "apollo-server-express": "^2.9.9", + "apollo-server-express": "^2.9.12", "camelcase": "^5.3.1", "cors": "^2.8.5", "dotenv": "^8.2.0", @@ -20,15 +20,15 @@ "helmet": "^3.21.2", "mariadb": "^2.1.3", "multer": "^1.4.2", - "mysql2": "^2.0.0", - "nodemon": "^1.19.4", - "pg": "^7.12.1", + "mysql2": "^2.0.1", + "nodemon": "^2.0.1", + "pg": "^7.14.0", "sequelize": "^5.21.2", "sequelize-cli": "^5.5.1", "sqlite3": "^4.1.0", "uuid": "^3.3.3" }, "devDependencies": { - "eslint": "^6.6.0" + "eslint": "^6.7.1" } }