New version 2.1

README.md

@@ -1,4 +1,4 @@
-# Rdio Scanner v2.0
+# Rdio Scanner v2.1
 
 *Rdio Scanner* is a progressive web interface designed to resemble an old school radio scanner. It integrates all frontend / backend components to manage audio files from different sources.
 
@@ -8,7 +8,9 @@ For now, only [Trunk Recorder](https://github.com/robotastic/trunk-recorder) sof
 
 ## What's new in this version
 
-The version 2.0 of *Rdio Scanner* is a major rewrite in which Meteor has been replaced by [The Apollo Data Graph Platform](https://www.apollographql.com/) for its API. MongoDB is also replaced by [SQLite](https://www.sqlite.org/) to facilitate the entire installation process. It is still possible to use another database by changing [Sequelize ORM](https://sequelize.org/) settings accordingly.
+Version 2.1 focuses on various speed improvements for searching stored calls.
+
+Version 2.0 is a major rewrite in which Meteor has been replaced by [The Apollo Data Graph Platform](https://www.apollographql.com/) for its API. MongoDB is also replaced by [SQLite](https://www.sqlite.org/) to facilitate the entire installation process. It is still possible to use another database by changing [Sequelize ORM](https://sequelize.org/) settings accordingly.
 
 These changes bring many *performance benefits* to *Rdio scanner* and make it *much easier to install*.
 

client/package.json

@@ -1,6 +1,6 @@
 {
     "name": "rdio-scanner-client",
-    "version": "2.0.0",
+    "version": "2.1.0",
     "private": true,
     "scripts": {
         "build": "ng build --cross-origin use-credentials --prod $*",
@@ -38,7 +38,7 @@
         "@angular/cli": "~8.3.19",
         "@angular/compiler-cli": "~8.2.14",
         "@angular/language-service": "~8.2.14",
-        "@types/node": "~12.12.8",
+        "@types/node": "~12.12.14",
         "codelyzer": "^5.2.0",
         "ts-node": "~8.5.2",
         "tslint": "~5.20.1",

client/src/app/components/rdio-scanner/rdio-scanner-calls-query.service.ts

@@ -46,18 +46,16 @@ export class AppRdioScannerCallsQueryService extends Query<RdioScannerCallsQuery
     document = gql`
         query rdioScannerCalls(
             $date: Date
-            $first: Int
+            $limit: Int
-            $last: Int
+            $offset: Int
-            $skip: Int
             $sort: Int
             $system: Int
             $talkgroup: Int
         ) {
             rdioScannerCalls(
                 date: $date
-                first: $first
+                limit: $limit
-                last: $last
+                offset: $offset
-                skip: $skip
                 sort: $sort
                 system: $system
                 talkgroup: $talkgroup
@@ -67,21 +65,8 @@ export class AppRdioScannerCallsQueryService extends Query<RdioScannerCallsQuery
                 dateStop
                 results {
                     id
-                    emergency
                     freq
-                    freqList {
-                        errorCount
-                        freq
-                        len
-                        pos
-                        spikeCount
-                    }
                     startTime
-                    stopTime
-                    srcList {
-                        pos
-                        src
-                    }
                     system
                     talkgroup
                 }

client/src/app/components/rdio-scanner/rdio-scanner.component.html

@@ -76,7 +76,7 @@
                 <mat-progress-bar color="primary" [mode]="searchResultsPending ? 'query' : 'determinate'">
                 </mat-progress-bar>
                 <mat-paginator [disabled]="searchResultsPending" [length]="searchResultsCount" [hidePageSize]="true"
-                    pageSize="10" [showFirstLastButtons]="true">
+                    [pageSize]="searchResults.value.length" [showFirstLastButtons]="true">
                 </mat-paginator>
             </div>
             <form class="rdio-search-form" [formGroup]="searchForm" autocomplete="off">

client/src/app/components/rdio-scanner/rdio-scanner.component.ts

@@ -1,8 +1,7 @@
-import { AfterViewInit, Component, HostListener, NgZone, OnDestroy, OnInit, ViewChild } from '@angular/core';
+import { Component, HostListener, NgZone, OnDestroy, OnInit, ViewChild } from '@angular/core';
 import { FormBuilder, FormGroup } from '@angular/forms';
-import { MatPaginator, PageEvent } from '@angular/material/paginator';
+import { MatPaginator } from '@angular/material/paginator';
-import { MatTable, MatTableDataSource } from '@angular/material/table';
+import { BehaviorSubject, Subscription } from 'rxjs';
-import { Subscription } from 'rxjs';
 import { AppRdioScannerCallQueryService } from './rdio-scanner-call-query.service';
 import { AppRdioScannerCallSubscriptionService, RdioScannerCall } from './rdio-scanner-call-subscription.service';
 import { AppRdioScannerCallsQueryService } from './rdio-scanner-calls-query.service';
@@ -24,7 +23,7 @@ interface RdioScannerSelection {
     styleUrls: ['./rdio-scanner.component.scss'],
     templateUrl: './rdio-scanner.component.html',
 })
-export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit {
+export class AppRdioScannerComponent implements OnDestroy, OnInit {
     get call() { return this._call; }
     get callHistory() { return this._callHistory; }
     get callPrevious() { return this._callPrevious; }
@@ -82,7 +81,7 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit
     private _searchFormDateStart = '';
     private _searchFormDateStop = '';
     private _searchPanelOpened = false;
-    private _searchResults = new MatTableDataSource<RdioScannerCall>();
+    private _searchResults = new BehaviorSubject(new Array<RdioScannerCall>(10));
     private _searchResultsCount = 0;
     private _searchResultsPending = false;
     private _selection: RdioScannerSelection = {};
@@ -98,10 +97,12 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit
     private livefeedSubscription: Subscription;
     private searchFormSubscription: Subscription;
     private searchPaginatorSubscription: Subscription;
+    private searchResultsBuffer = new Array<RdioScannerCall>(200);
+    private searchResultsBufferFrom = 0;
+    private searchResultsBufferTo = this.searchResultsBuffer.length - 1;
     private systemsSubscription: Subscription;
 
     @ViewChild(MatPaginator, { static: true }) private matPaginator: MatPaginator;
-    @ViewChild(MatTable, { static: true }) private matTable: MatTable<RdioScannerCall>;
 
     constructor(
         private appRdioScannerCallQuery: AppRdioScannerCallQueryService,
@@ -282,19 +283,13 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit
     loadAndPlay(call: RdioScannerCall): void {
         this.appRdioScannerCallQuery.fetch({ id: call.id }).subscribe(({ data }) => {
             if (data.rdioScannerCall) {
-                Object.assign(call, data.rdioScannerCall);
+                Object.assign(call, this.transformCall(data.rdioScannerCall));
 
                 this.play(call);
             }
         });
     }
 
-    ngAfterViewInit(): void {
-        if (this.matPaginator instanceof MatPaginator) {
-            this.searchResults.paginator = this.matPaginator;
-        }
-    }
-
     ngOnDestroy(): void {
         this.unsubscribeLivefeed();
 
@@ -314,7 +309,6 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit
 
         this.subscribeSystems().then(() => {
             this.subscribeSearchForm();
-
             this.subscribeSearchPaginator();
         });
     }
@@ -430,7 +424,7 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit
 
     toggleSearchPanel(opened = !this.searchPanelOpened): void {
         if (opened) {
-            this.searchCalls();
+            this.loadStoredCalls();
         }
 
         this._searchPanelOpened = opened;
@@ -525,97 +519,70 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit
         }, this.selection || {});
     }
 
-    private searchCalls(paginator: {
+    private async loadStoredCalls(filters?: any): Promise<void> {
-        first?: boolean | number;
+        const limit = this.searchResultsBuffer.length;
-        last?: boolean | number;
-        skip?: number;
-    } = {}): void {
-        const options: {
-            date?: Date;
-            first?: number;
-            last?: number;
-            skip?: number;
-            sort?: number;
-            system?: number;
-            talkgroup?: number;
-        } = {};
 
-        const count = 200;
+        const pageFrom = this.matPaginator.pageIndex * this.matPaginator.pageSize;
 
-        const form = this.searchForm.value;
+        const pageTo = this.matPaginator.pageIndex * this.matPaginator.pageSize + this.matPaginator.pageSize - 1;
 
-        if (form.date instanceof Date) {
+        if (!this.searchResultsPending && (filters || !pageFrom ||
-            options.date = form.date;
+            pageFrom < this.searchResultsBufferFrom || pageTo > this.searchResultsBufferTo)) {
-        }
 
-        if (typeof paginator.first === 'boolean') {
+            filters = filters || this.searchForm.value;
-            options.first = count;
 
-        } else if (typeof paginator.first === 'number') {
+            this.searchResultsBufferFrom = Math.floor(pageFrom / limit) * limit;
-            options.first = paginator.first;
+            this.searchResultsBufferTo = this.searchResultsBufferFrom + limit - 1;
 
-        } else if (typeof paginator.last === 'boolean') {
+            const options: {
-            options.last = count;
+                date?: Date;
+                limit?: number;
+                offset?: number;
+                sort?: number;
+                system?: number;
+                talkgroup?: number;
+            } = {};
 
-        } else if (typeof paginator.last === 'number') {
+            if (filters.date instanceof Date) {
-            options.last = paginator.last;
+                options.date = filters.date;
-
-        } else {
-            options.first = count;
-        }
-
-        if (typeof paginator.skip === 'number' && paginator.skip > 0) {
-            options.skip = paginator.skip;
-        }
-
-        if (typeof form.sort === 'number') {
-            options.sort = form.sort;
-        }
-
-        if (typeof form.system === 'number' && form.system !== -1) {
-            options.system = form.system;
-        }
-
-        if (typeof form.talkgroup === 'number' && form.talkgroup !== -1) {
-            options.talkgroup = form.talkgroup;
-        }
-
-        this._searchResultsPending = true;
-
-        this.searchForm.disable();
-
-        this.appRdioScannerCallsQuery.fetch(options, { fetchPolicy: 'no-cache' }).subscribe(({ data }) => {
-            const results = data.rdioScannerCalls.results.map((call) => this.transformCall(call));
-
-            if (this.searchResults.data.length < data.rdioScannerCalls.count) {
-                const ar = new Array<RdioScannerCall>(data.rdioScannerCalls.count - this.searchResults.data.length);
-
-                if (options.sort < 0) {
-                    this.searchResults.data.unshift(...ar);
-                } else if (options.sort > 0) {
-                    this.searchResults.data.push(...ar);
-                }
             }
 
-            if (options.last) {
+            if (limit) {
-                this.searchResults.data.splice((options.skip || this.searchResults.data.length) - results.length,
+                options.limit = limit;
-                    results.length, ...results);
-
-            } else {
-                this.searchResults.data.splice(options.skip || 0, results.length, ...results);
             }
 
-            this._searchResultsCount = data.rdioScannerCalls.count;
+            if (this.searchResultsBufferFrom) {
-            this._searchFormDateStart = data.rdioScannerCalls.dateStart;
+                options.offset = this.searchResultsBufferFrom;
-            this._searchFormDateStop = data.rdioScannerCalls.dateStop;
+            }
 
-            this.matTable.renderRows();
+            if (filters.sort < 0) {
-            this.searchResults.data = this.searchResults.data.slice(); // because renderRow() doesn't want to work
+                options.sort = -1;
+            }
 
-            this.searchForm.enable();
+            if (filters.system >= 0) {
+                options.system = filters.system;
+            }
+
+            if (filters.talkgroup >= 0) {
+                options.talkgroup = filters.talkgroup;
+            }
+
+            this._searchResultsPending = true;
+
+            const query = await this.appRdioScannerCallsQuery.fetch(options, { fetchPolicy: 'no-cache' }).toPromise();
+
+            for (let i = 0; i < limit; i++) {
+                this.searchResultsBuffer[i] = this.transformCall(query.data.rdioScannerCalls.results[i]);
+            }
+
+            this._searchResultsCount = query.data.rdioScannerCalls.count;
+            this._searchFormDateStart = query.data.rdioScannerCalls.dateStart;
+            this._searchFormDateStop = query.data.rdioScannerCalls.dateStop;
 
             this._searchResultsPending = false;
-        });
+        }
+
+        this.searchResults.next(this.searchResultsBuffer.slice(pageFrom % limit, pageTo % limit + 1));
     }
 
     private async subscribeLivefeed(): Promise<void> {
@@ -632,23 +599,17 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit
 
         if (!this.searchFormSubscription) {
             this.searchFormSubscription = this.searchForm.valueChanges.subscribe((value) => {
-                if (!this.searchResultsPending) {
+                if (!Object.keys(value).every((key) => value[key] === lastValue[key])) {
-                    if (!Object.keys(value).every((key) => value[key] === lastValue[key])) {
+                    lastValue = value;
-                        if (value.system !== lastValue.system) {
-                            this.searchForm.get('talkgroup').setValue(-1, { emitEvent: false });
-                        }
 
-                        lastValue = this.searchForm.value;
+                    if (value.system !== lastValue.system && value.talkgroup !== -1) {
-
+                        lastValue.talkgroup = -1;
-                        this.searchResults.data.splice(0, this.searchResults.data.length,
+                        this.searchForm.get('talkgroup').reset(lastValue.talkgroup);
-                            ...new Array<RdioScannerCall>(this.matPaginator.pageSize));
-
-                        this.searchCalls();
-
-                        if (this.matPaginator.pageIndex > 0) {
-                            this.matPaginator.firstPage();
-                        }
                     }
+
+                    this.matPaginator.firstPage();
+
+                    this.loadStoredCalls(value);
                 }
             });
         }
@@ -656,43 +617,7 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit
 
     private subscribeSearchPaginator(): void {
         if (!this.searchPaginatorSubscription) {
-            this.searchPaginatorSubscription = this.matPaginator.page.subscribe((event: PageEvent) => {
+            this.searchPaginatorSubscription = this.matPaginator.page.subscribe(() => this.loadStoredCalls());
-                const sortOrder = this.searchForm.get('sort').value;
-
-                if (sortOrder < 0 && event.pageIndex === 0) {
-                    this.searchCalls({ first: true });
-
-                } else if (sortOrder > 0 && event.pageIndex >= event.length / event.pageSize - 1) {
-                    this.searchCalls({ last: true });
-
-                } else if (event.length - event.pageIndex * event.pageSize < event.pageSize) {
-                    const needMore = !this.searchResults.data
-                        .slice(event.pageIndex * event.pageSize, event.pageIndex * event.pageSize + event.pageSize)
-                        .every((call) => call);
-
-                    if (needMore) {
-                        this.searchCalls({ last: true, skip: event.length });
-                    }
-
-                } else if (event.pageIndex > event.previousPageIndex) {
-                    const needMore = !this.searchResults.data
-                        .slice(event.pageIndex * event.pageSize, event.pageIndex * event.pageSize + event.pageSize)
-                        .every((call) => call);
-
-                    if (needMore) {
-                        this.searchCalls({ first: true, skip: event.pageIndex * event.pageSize });
-                    }
-
-                } else if (event.pageIndex < event.previousPageIndex) {
-                    const needMore = !this.searchResults.data
-                        .slice(event.pageIndex * event.pageSize - 1, event.pageIndex * event.pageSize + event.pageSize - 1)
-                        .every((call) => call);
-
-                    if (needMore) {
-                        this.searchCalls({ last: true, skip: event.pageIndex * event.pageSize + event.pageSize - 1 });
-                    }
-                }
-            });
         }
     }
 
@@ -707,11 +632,13 @@ export class AppRdioScannerComponent implements AfterViewInit, OnDestroy, OnInit
     }
 
     private transformCall(call: RdioScannerCall): RdioScannerCall {
-        call.systemData = (call && this.systems
+        if (call) {
-            .find((system: RdioScannerSystem) => system.system === call.system)) || {};
+            call.systemData = (call && this.systems
+                .find((system: RdioScannerSystem) => system.system === call.system)) || {};
 
-        call.talkgroupData = (call && call.systemData && Array.isArray(call.systemData.talkgroups) && call.systemData.talkgroups
+            call.talkgroupData = (call && call.systemData && Array.isArray(call.systemData.talkgroups) && call.systemData.talkgroups
-            .find((talkgroup: RdioScannerTalkgroup) => talkgroup.dec === call.talkgroup)) || {};
+                .find((talkgroup: RdioScannerTalkgroup) => talkgroup.dec === call.talkgroup)) || {};
+        }
 
         return call;
     }

package.json

@@ -1,6 +1,6 @@
 {
     "name": "rdio-scanner",
-    "version": "2.0.0",
+    "version": "2.1.0",
     "private": true,
     "main": "run.js",
     "scripts": {

server/graphql/datasources/rdio-scanner-call.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const { DataSource } = require('apollo-datasource');
-const { getPageRange } = require('../../helpers/paginator');
 const { callReducer } = require('../../helpers/rdio-scanner');
 
 class RdioScannerSystem extends DataSource {
@@ -21,7 +20,7 @@ class RdioScannerSystem extends DataSource {
         return callReducer(result.dataValues);
     }
 
-    async getCalls({ date, system, talkgroup }, { first, last, skip, sort }) {
+    async getCalls({ date, system, talkgroup }, { limit, offset, sort }) {
         const Op = this.store.Sequelize.Op;
 
         const attributes = {
@@ -59,7 +58,8 @@ class RdioScannerSystem extends DataSource {
 
         const count = await this.store.rdioScannerCall.count({ where });
 
-        const { limit, offset } = getPageRange({ count, first, last, skip });
+        limit = typeof limit === 'number' ? limit : 100;
+        offset = typeof offset === 'number' ? offset : 0;
 
         const calls = await this.store.rdioScannerCall.findAll({ attributes, limit, offset, order, where });
 

server/graphql/queries/rdio-scanner-calls.gql

@@ -1,8 +1,7 @@
 rdioScannerCalls(
     date: Date
-    first: Int
+    limit: Int
-    last: Int
+    offset: Int
-    skip: Int
     sort: Int
     system: Int
     talkgroup: Int

server/graphql/queries/rdio-scanner-calls.js

@@ -1,7 +1,7 @@
 'use strict';
 
-module.exports = () => async (_, { date, first, last, skip, sort, system, talkgroup }, { dataSources }) => {
+module.exports = () => async (_, { date, limit, offset, sort, system, talkgroup }, { dataSources }) => {
-    const { count, dateStart, dateStop, results } = await dataSources.rdioScannerCall.getCalls({ date, system, talkgroup }, { first, last, skip, sort });
+    const { count, dateStart, dateStop, results } = await dataSources.rdioScannerCall.getCalls({ date, system, talkgroup }, { limit, offset, sort });
 
     return { count, dateStart, dateStop, results };
 };

server/helpers/paginator.js

@@ -1,31 +0,0 @@
-'use strict';
-
-function getPageRange({ count, first, last, skip }) {
-    const page = {
-        default: 100,
-        max: 1000,
-    };
-
-    let limit;
-
-    let offset;
-
-    first = typeof first === 'number' && first > 0 ? Math.min(page.max, first) : typeof last !== 'number' ? page.default : null;
-
-    last = typeof last === 'number' && last > 0 ? Math.min(page.max, last) : null;
-
-    skip = typeof skip === 'number' && skip > 0 ? skip : 0;
-
-    if (typeof first === 'number') {
-        limit = Math.max(1, Math.min(count, skip + first) - skip);
-        offset = Math.min(count - 1, skip);
-
-    } else {
-        limit = count - Math.max(0, Math.min(count, skip) - last);
-        offset = Math.max(0, Math.max(count, skip) - last);
-    }
-
-    return { limit, offset };
-}
-
-module.exports = { getPageRange };

server/migrations/20191028144433-create-rdio-scanner-system.js

@@ -59,5 +59,5 @@ module.exports = {
 
             throw err;
         }
-    }
+    },
 };

server/migrations/20191029092201-create-rdio-scanner-call.js

@@ -87,5 +87,5 @@ module.exports = {
 
             throw err;
         }
-    }
+    },
 };

server/migrations/20191126135515-optimize-rdio-scanner-calls.js

@@ -0,0 +1,22 @@
+'use strict';
+
+module.exports = {
+
+    up: async (queryInterface, Sequelize) => {
+        const transaction = await queryInterface.sequelize.transaction();
+
+        try {
+            await queryInterface.removeIndex('rdioScannerCalls', ['system'], { transaction });
+
+            await queryInterface.removeIndex('rdioScannerCalls', ['talkgroup'], { transaction });
+
+            await transaction.commit();
+
+        } catch (err) {
+            await transaction.rollback();
+
+            throw err;
+        }
+    },
+};
+

server/package.json

@@ -1,6 +1,6 @@
 {
     "name": "rdio-scanner-server",
-    "version": "2.0.0",
+    "version": "2.1.0",
     "private": true,
     "main": "index.js",
     "scripts": {
@@ -11,7 +11,7 @@
     "license": "LICENSE",
     "dependencies": {
         "apollo-datasource": "^0.6.3",
-        "apollo-server-express": "^2.9.9",
+        "apollo-server-express": "^2.9.12",
         "camelcase": "^5.3.1",
         "cors": "^2.8.5",
         "dotenv": "^8.2.0",
@@ -20,15 +20,15 @@
         "helmet": "^3.21.2",
         "mariadb": "^2.1.3",
         "multer": "^1.4.2",
-        "mysql2": "^2.0.0",
+        "mysql2": "^2.0.1",
-        "nodemon": "^1.19.4",
+        "nodemon": "^2.0.1",
-        "pg": "^7.12.1",
+        "pg": "^7.14.0",
         "sequelize": "^5.21.2",
         "sequelize-cli": "^5.5.1",
         "sqlite3": "^4.1.0",
         "uuid": "^3.3.3"
     },
     "devDependencies": {
-        "eslint": "^6.6.0"
+        "eslint": "^6.7.1"
     }
 }