Added backend flask server

dd-code-api/README.md

+[[_TOC_]]
+
+### 1. Installation procedure
+
+1. delete any dd_venv folder  
+   ```rm -rf dd_venv```
+1. create dd_venv   
+   ```python3 -m venv dd_venv```
+1. change to dd_venv virtual environment   
+   ```source dd_venv/bin/activate```
+1. install the requirements  
+   ```python3 -m pip install -r requirements.txt```
+
+### 2. Setup the mongodb configuration in .env file
+```shell
+export MONGO_HOST=localhost
+export MONGO_PORT=27017
+export MONGO_DBNAME=ddcode
+
+export SRC_DIR="$PWD/src"
+```
+
+### 3. Run the server
+```shell
+source .env
+flask run --host=0.0.0.0 --port=5001
+```
+
+### 4. Leaving the virtual environment
+```shell
+deactivate
+```
+
+## Development
+
+To develop REST API accessed to wiki.js, use the graphql playground.
+* http://ddcode-srv1:3000/graphql

dd-code-api/app.py

+from flask import Flask, jsonify
+from flask_cors import CORS
+
+
+from models.species import get_all_species
+from models.stat import get_all_proteins, get_all_condensates
+from models.stat import get_protein_histogram, get_condensates_histogram, get_experiments_per_condensate
+from models.proteins import ProteinModel
+from models.condensates import CondensateModel
+from models.wiki import get_main_content
+from view_helpers import details_view, list_view
+
+app = Flask(__name__)
+CORS(app)
+
+protein_model = ProteinModel()
+condensate_model = CondensateModel()
+
+
+@app.route('/condensates/<unique_name>', methods=['GET'])
+def condensates_details(unique_name):
+    return details_view(condensate_model, unique_name)
+
+
+@app.route('/proteins/<uniprot_id>', methods=['GET'])
+def proteins_details(uniprot_id):
+    return details_view(protein_model, uniprot_id)
+
+
+@app.route('/condensates', methods=['GET'])
+def condensates_list():
+    return list_view(condensate_model)
+
+
+@app.route('/proteins', methods=['GET'])
+def proteins_list():
+    return list_view(protein_model)
+
+
+@app.route('/species', methods=['GET'])
+def species_list():
+    response = {
+        'data': get_all_species()
+    }
+    return jsonify(response)
+
+
+@app.route('/stats', methods=['GET'])
+def stats():
+    response = {
+        'protein_data': get_all_proteins(),
+        'condensate_data': get_all_condensates(),
+        'protein_hist': get_protein_histogram(),
+        'condensate_hist': get_condensates_histogram(),
+        'experiments': get_experiments_per_condensate()
+    }
+    return jsonify(response)
+
+
+@app.route('/wiki', methods=['GET'])
+def wiki():
+    return get_main_content()
+
+
+if __name__ == '__main__':
+    app.run(debug=True)

dd-code-api/models/condensates.py

+from models.db import db_conn
+from models.model import Model
+import models.proteins
+import models.experiments
+
+
+class CondensateModel(Model):
+    def __init__(self):
+        self.collection_name = "condensates"
+        self.allowed_params = {
+            'query': str,
+            'proteins': str,
+            'species_tax_id': str,
+            'page': int,
+            'size': int,
+        }
+        self.unique_key = "unique_name"
+        self.attributes = [
+            'name',
+            'unique_name',
+            'species_tax_id',
+            'species_name',
+            'proteins',
+            'data_sources'
+        ]
+        self.list_attributes = {
+            'name': True,
+            'unique_name': True,
+            'species_tax_id': True,
+            'species_name': True,
+            'proteins': True
+        }
+
+    def fetch_record(self, unique_id):
+        """
+        Overriding base function for fetch record.
+        Add custom and foreign key fields like "proteins"
+
+        :param(str) unique_id: Unique ID of the condensate
+        :return(dict):
+        """
+        record = self.serialize_record(db_conn[self.collection_name].find_one({self.unique_key: unique_id}))
+        if not record:
+            return
+        if not record.get('proteins'):
+            return record
+        record['proteins'] = models.proteins.ProteinModel().fetch_by_unique_keys(record.get('proteins'))
+        _, record['experiments'] = models.experiments.ExperimentModel().fetch_all(condensate_id=record.get('unique_name'))
+
+        return record
+

dd-code-api/models/db.py

+import settings
+from pymongo import MongoClient
+
+
+_client = MongoClient(
+    host=settings.MONGO_HOST,
+    port=int(settings.MONGO_PORT),
+    username=settings.MONGO_USER,
+    password=settings.MONGO_PASS
+)
+
+if not settings.MONGO_DBNAME:
+    raise NameError("Env variable MONGO_DBNAME might be empty")
+db_conn = _client[settings.MONGO_DBNAME]

dd-code-api/models/experiments.py

+from models.model import Model
+
+
+class ExperimentModel(Model):
+    def __init__(self):
+        self.collection_name = "experiments"
+        self.allowed_params = {
+            'condensate_id': str
+        }
+        self.unique_key = "exp_id"
+        self.attributes = [
+            'exp_id',
+            'temperature',
+            'detection_method',
+            'is_phase_separated',
+            'ph_value',
+            'salts',
+            'publication_link'
+        ]

dd-code-api/models/model.py

+from models.db import db_conn
+import settings
+
+class Model():
+    def __init__(self):
+        self.collection_name = None
+        self.allowed_params = None
+        self.unique_key = None
+        self.attributes = None
+        self.list_attributes = None
+
+    def fetch_by_unique_keys(self, unique_key_values):
+        """
+        Fetch multiple records by unique key identifier.
+
+        :param(list of str) unique_key_values: List of unique values
+        :return(int, list of dicts): List of record dicts
+        """
+        query_dict = {
+            self.unique_key: {'$in': unique_key_values}
+        }
+        results = db_conn[self.collection_name].find(filter=query_dict)
+        serialized_records = [self.serialize_record(rec) for rec in results]
+        return serialized_records
+
+    def fetch_all(self, **kwargs):
+        """
+        Fetch multiple pages of records matching query parameter and sorted as well
+
+        :param(dict) kwargs: Key-value pair for forming search query
+            Keys could any of the "allowed_params" or any of the following ones:
+                1. sort: Comma-separated string for attributes preceded by + or -
+                2. page: Page number (Pagination)
+                3. size: Size of pages (Pagination)
+                4. query: String for text search
+        :return(list of dict):
+        """
+        query_dict = {}
+
+        # Create usual filter parameters
+        for key, value in kwargs.items():
+            if key not in self.allowed_params:
+                raise AttributeError("{} is not an allowed param".format(key))
+            # These 4 to be treated separately
+            if key in ['page', 'size', 'query', 'sort']:
+                continue
+            query_dict[key] = self.allowed_params[key](value)
+        # Query Text Search
+        if 'query' in kwargs:
+            query_dict['$text'] = {'$search': kwargs['query']}
+
+        # print(query_dict)
+        if 'species_tax_id' in kwargs:
+            if query_dict['species_tax_id'] == 'all':
+                del query_dict['species_tax_id']
+            elif query_dict['species_tax_id'] == 'null':
+                query_dict['species_tax_id'] = None
+            results = db_conn[self.collection_name].find(query_dict, self.list_attributes)
+        else:
+            results = db_conn[self.collection_name].find(query_dict)
+
+        total_count = db_conn[self.collection_name].count(query_dict)
+        #results = db_conn[self.collection_name].find(query_dict)
+        #total_count = db_conn[self.collection_name].count(query_dict)
+        #print(query_dict)
+        # print(total_count)
+
+        # Pagination stuff
+        # prev_page_no = None
+        # next_page_no = None
+        # if total_count > 0 and ('page' in kwargs or 'size' in kwargs):
+        #     # print(kwargs)
+        #     page_no = int(kwargs.get('page', 1))
+        #     size = int(kwargs.get('size', settings.DEFAULT_PAGE_SIZE))
+        #     skip_value = (page_no - 1) * size
+        #     if skip_value >= total_count:
+        #         raise ValueError('Value for page_no and size exceeded total count {}'.format(total_count))
+        #     if page_no > 1:
+        #         prev_page_no = page_no - 1
+        #     if skip_value + size < total_count:
+        #         next_page_no = page_no + 1
+        #     results = results.skip(skip_value).limit(size)
+
+        # Sorting
+        # sort = []
+        # if 'sort' in kwargs:
+        #     for field in kwargs['sort'].split(','):
+        #         direction = 1
+        #         if field.startswith('-'):
+        #             field = field[1:]
+        #             direction = -1
+        #         sort.append((field, direction))
+        #     results = results.sort(sort)
+
+        serialized_records = [self.serialize_record(rec) for rec in results]
+
+        # return total_count, serialized_records, prev_page_no, next_page_no
+        return total_count, serialized_records
+
+    def fetch_record(self, unique_id):
+        """
+        Fetch single record by unique ID
+
+        :param(str) unique_id:
+        :return(dict):
+        """
+        return self.serialize_record(db_conn[self.collection_name].find_one({self.unique_key: unique_id}))
+
+    def serialize_record(self, record):
+        """
+        Serialize database object to dictionary
+        Only add allowed attributes
+
+        :param(dict) record:
+        :return(dict):
+        """
+        response = {}
+        for key in self.attributes:
+            response[key] = record.get(key)
+        return response

dd-code-api/models/proteins.py

+from models.model import Model
+from models.db import db_conn
+import models.ptms
+import models.condensates
+
+
+class ProteinModel(Model):
+    def __init__(self):
+        self.collection_name = "proteins"
+        self.allowed_params = {
+            'query': str,
+            'species_tax_id': str,
+            'ensembl_id': str,
+            'ensembl_gene_id': str,
+            'page': int,
+            'size': int,
+        }
+        self.unique_key = "uniprot_id"
+        self.attributes = [
+            'name',
+            'species_tax_id',
+            'species_name',
+            'sequence',
+            'uniprot_id',
+            'uniprot_readable_id',
+            'gene_name',
+            'ensembl_id',
+            'ensembl_gene_id',
+            'primary_data_source',
+            'sequence',
+            'antibody_link',
+            'llps_ptms',
+            'condensates',
+            'function',
+            'iupred_score'
+        ]
+        self.list_attributes = {
+            'name': True,
+            'gene_name': True,
+            'species_tax_id': True,
+            'species_name': True,
+            'uniprot_id': True,
+            'uniprot_readable_id': True
+        }
+
+    def fetch_record(self, unique_id):
+        """
+        Overriding base function for fetch record.
+        Add custom and foreign key fields like "antibody_link" and "llps_ptms"
+
+        :param(str) unique_id:
+        :return(dict):
+        """
+        record = self.serialize_record(db_conn[self.collection_name].find_one({self.unique_key: unique_id}))
+        if not record:
+            return
+        if not record.get('llps_ptms'):
+            return record
+        record['llps_ptms'] = models.ptms.PtmModel().fetch_by_unique_keys(record.get('llps_ptms'))
+        _, record['condensates'] = models.condensates.CondensateModel().fetch_all(proteins=record.get('uniprot_id'))
+        record['antibody_link'] = "https://www.proteinatlas.org/{}/antibody".format(unique_id)
+        return record

dd-code-api/models/ptms.py

+from models.model import Model
+
+
+class PtmModel(Model):
+    def __init__(self):
+        self.collection_name = "post_translational_modifications"
+        self.unique_key = "unique_name"
+        self.attributes = [
+            'unique_name',
+            'name',
+            'position',
+            'enzyme',
+            'effect_type',
+            'effect_description'
+        ]

dd-code-api/models/species.py

+from models.db import db_conn
+
+
+def get_all_species():
+    condensates_coll = db_conn['condensates']
+    aggregate_query = [
+        {
+            '$group': {
+                '_id': "$species_name",
+                'tax_id': {
+                    '$first': "$species_tax_id"
+                },
+                'count': {
+                    '$sum': 1
+                }
+            }
+        },
+        {
+            '$sort': {
+                'count': -1
+            }
+        }
+    ]
+    response = condensates_coll.aggregate(pipeline=aggregate_query)
+    return list(response)

dd-code-api/models/stat.py

+from models.db import db_conn
+
+
+def get_all_proteins():
+    proteins_coll = db_conn['proteins']
+    aggregate_query = [
+        {
+            '$group': {
+                '_id': "$species_name",
+                'total': {'$sum': 1}
+            }
+        },
+        {
+            '$sort': {'total': -1}
+        }
+    ]
+    response = proteins_coll.aggregate(aggregate_query)
+    return list(response)
+
+
+def get_all_condensates():
+    condensates_coll = db_conn['condensates']
+    aggregate_query = [
+        {
+            '$project': {
+                '_id': 1,
+                'species_name': {'$ifNull': ["$species_name", "Chimeras"]}
+            }
+        },
+        {
+            '$group': {
+                '_id': "$species_name",
+                'total': {'$sum': 1}
+            }
+        },
+        {
+            '$sort': {'total': -1}
+        }
+    ]
+    response = condensates_coll.aggregate(aggregate_query)
+    return list(response)
+
+
+def get_protein_histogram():
+    proteins_coll = db_conn['proteins']
+    aggregate_query = [
+        {
+            '$lookup': {
+                'from': 'condensates',
+                'localField': 'uniprot_id',
+                'foreignField': 'proteins',
+                'as': 'condensates'
+            }
+        },
+        {
+            '$match': {
+                'uniprot_id': {
+                    '$exists': True,
+                    '$ne': None
+                }
+            }
+        },
+        {
+            '$project': {
+                'uniprot_id': 1,
+                'size': {
+                    '$cond': {
+                        'if': {'$isArray': '$condensates'},
+                        'then': {'$size': '$condensates'},
+                        'else': 0
+                    }
+                }
+            }
+        },
+        {
+            '$group': {
+                '_id': '$size',
+                'uniprot_id': {'$push': '$uniprot_id'},
+                'total': {'$sum': 1}
+            }
+        },
+        {
+            '$sort': {'_id': 1}
+        }
+    ]
+    response = proteins_coll.aggregate(aggregate_query)
+    return list(response)
+
+
+def get_condensates_histogram():
+    condensates_coll = db_conn['condensates']
+    aggregate_query = [
+        {
+            '$project': {
+                'unique_name': 1,
+                'size': {
+                    '$cond': {
+                        'if': {'$isArray': '$proteins'},
+                        'then': {'$size': '$proteins'},
+                        'else': 0
+                    }
+                }
+            }
+        },
+        {
+            '$group': {
+                '_id': '$size',
+                'condensate': {'$push': '$unique_name'},
+                'total': {'$sum': 1}
+            }
+        },
+        {
+            '$sort': {'_id': 1}
+        }
+    ]
+    response = condensates_coll.aggregate(aggregate_query)
+    return list(response)
+
+
+def get_experiments_per_condensate():
+    experiments_coll = db_conn['experiments']
+    aggregate_query = [
+        {
+            '$group': {
+                '_id': '$condensate_id',
+                'total': {'$sum': 1}
+            }
+        },
+        {
+            '$sort': {'total': -1}
+        }
+    ]
+    response = experiments_coll.aggregate(aggregate_query)
+    return list(response)
+

dd-code-api/models/wiki.py

+import requests
+import settings
+
+url = 'http://ddcode-srv1:3000/graphql'
+
+
+def get_main_content():
+    json = {'query': '''
+    query {
+        pages {
+         single(id: 1){
+           title,
+           content,
+         }
+        }
+    }
+    '''}
+
+    headers = {'Authorization': 'Bearer %s' % settings.WIKI_API_TOKEN}
+
+    r = requests.post(url=url, json=json, headers=headers)
+    return r.text

dd-code-api/requirements.txt

+flask
+flask-restful
+flask-cors
+requests
+pymongo
\ No newline at end of file

dd-code-api/run.sh

+source ./dd_venv/bin/activate
+source .env
+flask run --host=0.0.0.0 --port=5001
+

dd-code-api/settings.py

+import os
+
+SRC_DIR = os.environ.get('SRC_DIR')
+
+# Database configurations
+MONGO_HOST = os.environ.get('MONGO_HOST')
+MONGO_PORT = os.environ.get('MONGO_PORT')
+MONGO_USER = os.environ.get('MONGO_USER')
+MONGO_PASS = os.environ.get('MONGO_PASS')
+MONGO_DBNAME = os.environ.get('MONGO_DBNAME')
+
+# Wiki API
+WIKI_API_TOKEN = os.environ.get('WIKI_API_TOKEN')
+
+DEFAULT_PAGE_SIZE = 10

dd-code-api/view_helpers.py

+from urllib.parse import urlencode
+from flask import jsonify, request
+
+
+def details_view(model, unique_id):
+    """
+    Generic helper function for rendering details API
+
+    :param(model.Model) model: Model object of the respective entity
+    :param(str) unique_id: Unique ID of the resource to be fetched
+    :return: JSON object and HTTP Status Code
+    """
+    record = model.fetch_record(unique_id)
+    if not record:
+        response = {
+            'code': 'NOT FOUND',
+            'message': 'Resource with ID {} not found in our database'.format(unique_id)
+        }
+        status_code = 404
+    else:
+        response = {
+            'data': record
+        }
+        status_code = 200
+
+    return jsonify(response), status_code
+
+
+def list_view(model):
+    """
+    Helper function for list view APIs
+
+    :param(model.Model) model: Model object of the respective entity
+    :return: JSON object and HTTP Status Code
+    """
+    kwargs = {}
+    for key in request.args.keys():
+        if key not in model.allowed_params:
+            response = {
+                'code': 'BAD REQUEST',
+                'message': 'Query Param {} not allowed'.format(key)
+            }
+            return jsonify(response), 400
+        kwargs[key] = request.args.get(key)
+    if 'page' not in kwargs:
+        kwargs['page'] = 1
+    # count, records, prev_page_no, next_page_no = model.fetch_all(**kwargs)
+    count, records = model.fetch_all(**kwargs)
+    serialized_records = [model.serialize_record(rec) for rec in records]
+
+    # filters = dict(request.args.deepcopy())
+    # filters['page'] = next_page_no
+    # next_page_url = None if not next_page_no else "{}?{}".format(request.base_url, urlencode(filters))
+    # filters['page'] = prev_page_no
+    # prev_page_url = None if not prev_page_no else "{}?{}".format(request.base_url, urlencode(filters))
+
+    response = {
+        'data': serialized_records,
+        'count': count
+        # 'next_page': next_page_url,
+        # 'prev_page': prev_page_url
+    }
+
+    return jsonify(response), 200