Merge pull request #26 from demokratie-live/sprint#6/memory-issue

Sprint#6/memory issue

Author: ulfgebhardt

.gitignore

@@ -1,2 +1,3 @@
 node_modules/
-.env
+*.env
+*.log

Dockerfile

@@ -1,14 +1,9 @@
-FROM node:carbon
+FROM node:latest
 
 WORKDIR /app
 
-COPY package*.json ./
-
-RUN yarn
-
-# Bundle app source
 COPY . .
 
-# EXPOSE 8080
+RUN yarn install
 
-CMD [ "yarn", "start" ]
+ENTRYPOINT [ "yarn", "start" ]

package.json

@@ -6,11 +6,9 @@
   "license": "Apache2",
   "private": true,
   "scripts": {
-    "start": "nodemon --exec babel-node src/index.js",
+    "start": "babel-node src/index.js",
     "dev":
-      "nodemon -e js,graphql src/index.js --exec \"node --require dotenv/config --require babel-register\"",
-    "imp": "nodemon -e js,graphql --exec \"babel-node scripts/import.js\"",
-    "impAll": "nodemon -e js,graphql --exec \"babel-node scripts/importAll.js\"",
+      "nodemon -L -e js,graphql src/index.js --exec \"node --require dotenv/config --require babel-register\"",
     "lint": "eslint .",
     "clean": "rm -rf build && mkdir build",
     "build": "babel -d build src -s",

src/config/constants.js

@@ -1,10 +1,7 @@
 export default {
   PORT: process.env.PORT || 3000,
   db: {
-    development: {
-      app: 'mongodb://localhost/democracy_development',
-    },
-    url: process.env.DB || 'mongodb://localhost/democracy_development',
+    url: process.env.DB_URL || 'mongodb://localhost/democracy_development',
   },
   GRAPHIQL_PATH: '/graphiql',
   GRAPHQL_PATH: '/graphql',

src/graphql/client.js

@@ -4,12 +4,13 @@ import { InMemoryCache } from 'apollo-cache-inmemory';
 import fetch from 'node-fetch';
 import CONSTANTS from '../config/constants';
 
-const client = new ApolloClient({
+
+const createClient = () => new ApolloClient({
   link: new HttpLink({
     // ssrMode: true,
     uri: CONSTANTS.BUNDESTAGIO_SERVER_URL,
     fetch,
   }),
   cache: new InMemoryCache(),
 });
-export default client;
+export default createClient;

src/graphql/schemas/index.js

@@ -1,6 +1,7 @@
 import { fileLoader, mergeTypes } from 'merge-graphql-schemas';
 import path from 'path';
+import _ from 'lodash';
 
-const typesArray = fileLoader(path.join(__dirname, './'));
+const typesArray = _.uniq(fileLoader(path.join(__dirname, './')));
 
 export default mergeTypes(typesArray);

src/scripts/import.js

@@ -1,15 +1,17 @@
 import ProgressBar from 'cli-progress'; // eslint-disable-line
 import program from 'commander'; // eslint-disable-line
 
-import client from '../graphql/client';
+import createClient from '../graphql/client';
 import Procedure from '../models/Procedure';
 import getProcedures from '../graphql/queries/getProcedures';
 
 export default async (procedureIds) => {
+  const client = createClient();
   // Start Importing
   const { data: { procedures } } = await client.query({
     query: getProcedures,
     variables: { IDs: procedureIds },
+    fetchPolicy: 'network-only',
   });
   // Start Inserting
   const promises = await procedures.map(async (bIoProcedure) => {

src/scripts/webhook.js

@@ -1,36 +1,39 @@
 import importProcedures from './import';
 import getProcedureUpdates from '../graphql/queries/getProcedureUpdates';
-import client from '../graphql/client';
+import createClient from '../graphql/client';
 import ProcedureModel from '../models/Procedure';
 
 export default async (data) => {
+  const client = createClient();
+
   // Count local Data in groups
-  const groups = await ProcedureModel.aggregate([{
-    // Group by Period & Type
-    $group: {
-      _id: { period: '$period', type: '$type' },
-      count: { $sum: 1 },
+  const groups = await ProcedureModel.aggregate([
+    {
+      // Group by Period & Type
+      $group: {
+        _id: { period: '$period', type: '$type' },
+        count: { $sum: 1 },
+      },
+    },
+    {
+      // Group by Period
+      $group: {
+        _id: '$_id.period',
+        types: { $push: { type: '$_id.type', count: '$count' } },
+      },
     },
-  },
-  {
-    // Group by Period
-    $group: {
-      _id: '$_id.period',
-      types: { $push: { type: '$_id.type', count: '$count' } },
+    {
+      // Rename _id Field to period
+      $project: { _id: 0, period: '$_id', types: 1 },
     },
-  },
-  {
-    // Rename _id Field to period
-    $project: { _id: 0, period: '$_id', types: 1 },
-  }]);
+  ]);
 
   const update = [];
   await Promise.all(data.map(async (d) => {
     const period = parseInt(d.period, 10);
     const { type, countBefore, changedIds } = d.types.find(t => t.type === 'Gesetzgebung');
     const group = groups.find(c => c.period === period);
-    const localCount = group ? group.types
-      .find(ct => ct.type === type).count : 0;
+    const localCount = group ? group.types.find(ct => ct.type === type).count : 0;
     // Append Changed IDs
     update.concat(changedIds);
     // Compare Counts Remote & Local
@@ -40,10 +43,12 @@ export default async (data) => {
         query: getProcedureUpdates,
         variables: { period, type },
       });
-      // Find local Procedure Updates
-      const localProcedureUpdates = await ProcedureModel
-        .find({ period, type }, { procedureId: 1, lastUpdateDate: 1 });
-      // Compare
+        // Find local Procedure Updates
+      const localProcedureUpdates = await ProcedureModel.find(
+        { period, type },
+        { procedureId: 1, lastUpdateDate: 1 },
+      );
+        // Compare
       procedureUpdates.forEach((pu) => {
         const localData = localProcedureUpdates.find(ld => ld.procedureId === pu.procedureId);
         if (!localData || new Date(localData.lastUpdateDate) < new Date(pu.updatedAt)) {