Skip to content
Snippets Groups Projects
Commit cbe53c68 authored by Maxim Scheremetjew's avatar Maxim Scheremetjew
Browse files

chore, docs: Initial project setup.

parent d1b86d8f
No related branches found
No related tags found
No related merge requests found
# .coveragerc to control coverage.py
[run]
branch = True
omit =
# omit individual files
server.py
app/conftest.py
# omit test folders
*/tests/*
[report]
fail_under = 90.0
[XML]
output = codecov-output/coverage.xml
\ No newline at end of file
# Git
.git
.gitignore
# CI
.codeclimate.yml
.travis.yml
.taskcluster.yml
# Docker
docker-compose.yml
.docker
# Byte-compiled / optimized / DLL files
__pycache__/
*/__pycache__/
*/*/__pycache__/
*/*/*/__pycache__/
*.py[cod]
*/*.py[cod]
*/*/*.py[cod]
*/*/*/*.py[cod]
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.cache
nosetests.xml
coverage.xml
# Translations
*.mo
*.pot
# Django stuff:
*.log
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Virtual environment
.env/
.venv/
venv/
# PyCharm
.idea
# VS Code
.vscode
# Python mode for VIM
.ropeproject
*/.ropeproject
*/*/.ropeproject
*/*/*/.ropeproject
# Vim swap files
*.swp
*/*.swp
*/*/*.swp
*/*/*/*.swp
# Backup files
*.bak
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Mac spcific system files
.DS_Store
.DS_Store?
.directory
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
.pytest_cache/
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Log files:
*.log
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# IPython Notebook
.ipynb_checkpoints
# pyenv
.python-version
# Pycharm
.idea/
# VS Code
.vscode
# dotenv
.env
.out
# virtualenv
venv/
ENV/
# Static files
static/
# Media files
media/
#Docker-compose
docker-compose-local.yml
docker-compose.override.yml
# Database backups
.backups/
# Vim (copied https://github.com/github/gitignore/blob/master/Global/Vim.gitignore)
.sw[a-z]
Session.vim
.netrwhist
*~
tags
# Database dumps
*.dump
# Atom
.tags
output/
FROM python:3.11.3-slim-buster
ENV PYTHONUNBUFFERED=1
ENV DIRPATH=/var/levenshtein-distance-service
WORKDIR $DIRPATH
# Now, be a web server.
EXPOSE 8001
CMD ["python", "./server.py"]
Makefile 0 → 100644
include ./make/print.lib.mk
include ./make/dynamic-recipe.lib.mk
#------------------------------
# dynamic recipes
#
# marked with # dynamic
# A named recipe (eg. manage) will set CMD, and call it
# if it's the last recipe being called (eg. make manage).
# A recipe caught by .DEFAULT will add the recipe name
# (eg. changepassword) to a list of ARGS, and call CMD with
# the list of ARGS if it's the last recipe being called
# (eg. make manage changepassword youruser).
#
# NOTE: don't create any recipes with the same name as any
# dynamic recipe args (eg. changepassword)
#------------------------------
#------------------------------
# vars
#------------------------------
SHELL := /bin/bash
CMD := ""
POS_ARGS := ""
ARGS := ""
SSH_AUTH_SOCK_RSA_MOUNT_VOLUMES = -v ~/.ssh/id_rsa:/id_rsa:ro -v ~/.ssh/id_rsa.pub:/id_rsa.pub:ro
SSH_AUTH_SOCK_ED25519_MOUNT_VOLUMES = -v ~/.ssh/id_ed25519:/id_ed25519:ro -v ~/.ssh/id_ed25519.pub:/id_ed25519.pub:ro
DOCKER_RUN_ARGS_FOR_SSH_AUTH_SOCK_RSA := --entrypoint= --rm --tty --interactive --env SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock -v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock ${SSH_AUTH_SOCK_RSA_MOUNT_VOLUMES}
DOCKER_RUN_ARGS_FOR_SSH_AUTH_SOCK_ED25519 := --entrypoint= --rm --tty --interactive --env SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock -v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock ${SSH_AUTH_SOCK_ED25519_MOUNT_VOLUMES}
SERVICE_TAG :=mscheremetjew/levenshtein-distance-service
#------------------------------
# helpers
#------------------------------
COMMA := ,
#------------------------------
# help
#------------------------------
.PHONY: help
help:
$(call print_h1,"AVAILABLE","OPTIONS")
$(call print_space)
$(call print_h2,"code")
$(call print_options,"lint","Run code lint checks.")
$(call print_options,"format","Automatically format code where possible.")
$(call print_space)
$(call print_h2,"dependency")
$(call print_options,"pip-compile-rsa","Compile requirements.txt from requirements.in without upgrading the packages and build the images using RSA SSH key.")
$(call print_options,"pip-compile-ed-25519","Compile requirements.txt from requirements.in without upgrading the packages and build the images with ed-25519 SSH key.")
$(call print_space)
$(call print_h2,"test")
$(call print_options,"pytest","Run all pytests (takes args additional via ARGS=\"...\" eg. \`\`make pytest ARGS=\"entertainment/tests/ --reuse-db\"\`\` or \`\`make pytest ARGS=\"-m \'mark1 and not mark2\'\"\`\`).")
$(call print_options,"pytest-h","Show pytest help")
$(call print_space)
$(call print_h2,"dynamic recipes")
$(call print_h3,"accepts any number of additional positional args as well as --args via ARGS=\"...\"")
#------------------------------
# code
#------------------------------
.PHONY: lint
lint: build-services
$(call print_h1,"LINTING","CODE")
@docker-compose run --rm --entrypoint=sh distance_calculator -c "flake8"
@docker-compose run --rm --entrypoint=sh distance_calculator -c "isort --recursive --check-only"
@docker-compose run --rm --entrypoint=sh distance_calculator -c "black --check ./"
$(call print_h1,"LINTING","COMPLETE")
.PHONY: format
format: build-services
$(call print_h1,"FORMATTING","CODE")
@docker-compose run --rm --entrypoint=sh distance_calculator -c "isort --recursive --apply"
@docker-compose run --rm --entrypoint=sh distance_calculator -c "black ./"
$(call print_h1,"FORMATTING","COMPLETE")
#------------------------------
# docker helpers
#------------------------------
# runs docker compose with the provided args,
# and prints out the full command being run
define dockercompose
$(call print,"docker-compose $(1)")
@docker-compose $(1)
endef
# checks for 'local' command and if local config file exists,
# and if so runs docker-compose using the local file,
# if not prints a warning and runs with default config.
define dockercomposelocal
@$(eval TARGETING_LOCAL := $(if $(filter-out local,$(lastword $(MAKECMDGOALS))),,""))
@$(eval LOCAL_FILE_EXISTS := $(if $(wildcard $(FILE_PATH_DOCKER_COMPOSE_LOCAL)),"",))
@$(eval DOCKER_COMPOSE_FILE_ARG := $(if $(and $(TARGETING_LOCAL),$(LOCAL_FILE_EXISTS)),"-f $(FILE_PATH_DOCKER_COMPOSE_LOCAL) ",))
@$(if $(TARGETING_LOCAL),$(if $(LOCAL_FILE_EXISTS),,$(call print_warning,"Local config missing$(COMMA) please create: $(FILE_PATH_DOCKER_COMPOSE_LOCAL)$(COMMA) using default...")),)
$(call dockercompose,"$(DOCKER_COMPOSE_FILE_ARG)$(1)")
endef
define build-services
$(call print_h1,"BUILDING","IMAGES","FROM","SCRATCH")
@docker build --ssh default -t $(SERVICE_TAG) -f Dockerfile .
$(call print_h1,"IMAGES","BUILT","FROM","SCRATCH")
endef
#------------------------------
# docker
#------------------------------
.PHONY: build-services
build-services:
$(call build-services)
.PHONY: up
up: build-services
$(call print_h1,"LAUNCHING","ALL","DOCKER","CONTAINERS")
$(call dockercomposelocal,"up -d")
.PHONY: buildup
buildup: build-services
$(call print_h1,"REBUILDING","AND","LAUNCHING","ALL DOCKER CONTAINERS")
$(call dockercomposelocal,"up --build")
#------------------------------
# utility
#------------------------------
.PHONY: shell-distance_calculator
shell-distance_calculator:
$(call print_h1,"ENTERING","SHELL")
@docker-compose exec distance_calculator /bin/bash
#------------------------------
# dependency
#------------------------------
.PHONY: pip-compile-rsa pip-compile-ed-25519
pip-compile-rsa: build-services
$(call print_h1,"COMPILING","REQUIREMENTS")
@-docker run $(DOCKER_RUN_ARGS_FOR_SSH_AUTH_SOCK_RSA) -v ${PWD}:/var/levenshtein-distance-service/ $(SERVICE_TAG) sh -c "ssh-add /id_rsa && pip-compile --no-header --output-file=app/requirements.txt"
$(call build-services)
$(call print_h1,"REQUIREMENTS","COMPILED")
pip-compile-ed-25519: build-services
$(call print_h1,"COMPILING","REQUIREMENTS")
@-docker run $(DOCKER_RUN_ARGS_FOR_SSH_AUTH_SOCK_ED25519) -v ${PWD}:/var/levenshtein-distance-service/ $(SERVICE_TAG) sh -c "ssh-add /id_ed25519 && pip-compile --no-header --output-file=app/requirements.txt"
$(call build-services)
$(call print_h1,"REQUIREMENTS","COMPILED")
#------------------------------
# Q&A
#------------------------------
.PHONY: pytest
pytest:
$(call print_h1,"RUNNING","PYTEST","TESTS")
@$(eval CMD := "docker-compose run --rm --entrypoint=sh distance_calculator -c 'pytest -vv'")
$(call run_if_last,${CMD} ${POS_ARGS} ${ARGS})
.PHONY: pytest-h
pytest-h:
$(call print_h1,"SHOWING","PYTEST","HELP")
@docker-compose run --rm --entrypoint=sh distance_calculator -c "pytest --help"
#------------------------------
# dynamic functionality
#------------------------------
# adds recipe name (eg. changepassword) to POS_ARGS, calls CMD with ARGS and POS_ARGS if last
.DEFAULT:
@$(eval POS_ARGS += $@)
@$(eval ARGS += $(ARGS))
$(call run_if_last,${CMD} ${ARGS} ${POS_ARGS})
\ No newline at end of file
# levenshtein-distance-service
A web service for calculating the Levenshtein distance between two sequences
# levenshtein-distance-service
## Table of content
1. [Introduction](#introduction)
2. [Development](#development)
1. [Getting started](#getting-started)
2. [How to enable BuildKit?](#how-to-enable-buildkit)
1. [Enable the BuildKit in Docker Desktop](#enable-the-buildkit-in-docker-desktop)
2. [Enable the BuildKit in a fresh terminal](#enable-the-buildkit-in-a-fresh-terminal)
3. [On Linux machines, you may also need](#on-linux-machines-you-may-also-need)
3. [Building your Docker images](#building-your-docker-images)
4. [How to update project dependencies?](#how-to-update-project-dependencies)
5. [How to run services locally?](#how-to-run-services-locally)
3. [Testing and code formatting](#testing-and-code-formatting)
1. [How to run tests locally?](#how-to-run-tests-locally)
2. [How to format the code before pushing new changes to remote?](#how-to-format-the-code-before-pushing-new-changes-to-remote)
4. [Continuous integration/CI](#continuous-integration)
1. [Code coverage](#code-coverage)
## Introduction
A web service for calculating the Levenshtein distance between two sequences.
In order to compare and study proteins, scientists have devised many computational techniques and
algorithms. One such algorithm is the “Levenshtein distance”, which computes the distance between
two string-like sequences. It is used as a simple measure of similarity between two proteins.
The aim of this project is to develop a web application where a user can compute the Levenshtein distance between
two proteins.
## Development
### Getting started
...
### How to enable BuildKit?
This could be achieved in different ways.
#### Enable the BuildKit in Docker Desktop
![image info](docs/screenshot_docker_desktop_enable_buildkit.png)
#### Enable the BuildKit in a fresh terminal
```
export DOCKER_BUILDKIT=1
```
##### On Linux machines, you may also need:
```
export COMPOSE_DOCKER_CLI_BUILD=1
```
### Building your Docker images
```shell
docker build --ssh default . -f Dockerfile -t mscheremetjew/levenshtein-distance-service
docker-compose build
```
And then the service can be started as following:
```shell
docker-compose up
```
### How to update project dependencies?
For this project we are using pip-tools to manage all project dependencies. Apply changes to
requirements.in and run pip-compile, which generates an updated version of requirements.txt.
For rsa keys run...
```console
make pip-compile-rsa
```
For ed-25519 keys run...
```console
make pip-compile-ed-25519
```
### How to run services locally?
#### How to start all services defined in docker-compose file - for development only?
```shell
docker-compose up
```
OR use the following Makefile command:
```shell
make up
```
#### How to start services marked with specific profiles - for development only?
Certain services are marked with specific profiles. To start these services
you have to use the --profile option in your docker-compose command.
```shell
docker-compose [--profile prometheus] up
```
## Testing and code formatting
### How to run tests locally?
```console
make pytest
OR you can run specific test like this...
docker-compose run --rm --entrypoint=sh <service-name> -c "pytest app/tests/unit/test_*.py"
```
### How to format the code before pushing new changes to remote?
```console
make format
make lint
```
## Continuous integration
...
### Code coverage
For this project code coverage is set up.
\ No newline at end of file
"""Settings for the Levenshtein distance service."""
import logging.config
from environs import Env
logger = logging.getLogger(__name__)
env = Env()
def get_logging_config(env, loggers_to_silent=None):
"""Return the python logging configuration based on environment variables.
The log level for the given loggers_to_silent will be set to INFO.
Use this for loggers that at DEBUG level put too much entries that we never take care.
The log level for specific loggers can be more customized with the CUSTOM_LOGGING environment
variable, providing a list of logger and level in the form: CUSTOM_LOGGING=<logger>=<LEVEL>
This will overwrite the configuration set because of loggers_to_silent (so can be used to put
to DEBUG some of the silenced loggers).
Example:
CUSTOM_LOGGING=botocore=DEBUG,elasticsearch=WARNING
"""
log_handlers = env.list("LOG_HANDLERS", default=["console"])
logging_config = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {},
"handlers": {},
"loggers": {
"": {
"handlers": log_handlers,
"level": env.str("LOG_LEVEL", default=env.str("GENERAL_LOGGING_LEVEL", default="DEBUG")),
},
},
}
if "console" in log_handlers:
_update_logging_config(logging_config, _get_console_config(env))
if loggers_to_silent:
for logger in loggers_to_silent:
logging_config["loggers"][logger] = {
"level": "INFO",
}
for logger_config in env.list("CUSTOM_LOGGING", default=[]):
logger, level = logger_config.split("=")
logging_config["loggers"][logger] = {
"level": level,
}
return logging_config
def _update_logging_config(logging_config: Dict, config: Dict):
logging_config["formatters"].update(config.get("formatters", {}))
logging_config["handlers"].update(config.get("handlers", {}))
def _get_console_config(env: Env) -> Dict:
return {
"formatters": {
"simple": {
"format": "%(asctime)s.%(msecs)03d:%(levelname)s:%(name)s:%(message)s",
"datefmt": "%Y-%m-%d %H:%M:%S",
},
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"level": env.str("LOG_LEVEL", default=env.str("CONSOLE_LOGGING_HANDLER_MIN_LEVEL", default="WARNING")),
"formatter": "simple",
},
},
}
# The logging must to be the first thing to configure to get the logs raised here
LOGGING = get_logging_config(
env,
loggers_to_silent=[
# The following loggers put lot of DEBUG entries and we only care if there is some error
# to show the DEBUG level, use the CUSTOM_LOGGING env variable
"urllib3.util.retry",
"urllib3.connectionpool",
],
)
logging.config.dictConfig(LOGGING)
version: "3.1"
services:
distance_calculator:
stdin_open: true
build:
context: .
dockerfile: Dockerfile
command: adev runserver -p 8002
ports:
- "8002:8002"
environment:
# Logging
- CONSOLE_LOGGING_HANDLER_MIN_LEVEL=INFO
- CUSTOM_LOGGING
\ No newline at end of file
# run provided command if current recipe is last in goals
define run_if_last
@ARR=($(MAKECMDGOALS)); \
if [ "$@" == "$${ARR[ $${#ARR[@]} - 1 ]}" ]; \
then eval $1; \
fi
endef
MAKE_THEME := LIME
-include .env
BLACK := 232
LIGHTER_GREY := 246
LIGHT_GREY := 244
GREY := 243
DARK_GREY := 237
PLAIN := 255
BLUE := 4
GOLD := 214
LIGHTBLUE := 74
LIME := 106
PURPLE := 99
PINK := 219
RESET := "\033[0m"
FG = "\033[38;5;$(1)m"
FGDIM = "\033[2;38;5;$(1)m"
FGB = "\033[1;38;5;$(1)m"
BG = "\033[48;5;$(1)m"
define print_h1
@$(eval COLOR=$($(MAKE_THEME)))
@if [[ -n '$(4)' ]]; \
then printf '%b%b %s %b%b %s %b%b %s %b%b %s %b\n' $(call BG,$(DARK_GREY)) $(call FG,$(COLOR)) $(1) $(call BG,$(COLOR)) $(call FGB,$(BLACK)) $(2) $(call BG,$(DARK_GREY)) $(call FG,$(COLOR)) $(3) $(call BG,$(COLOR)) $(call FGB,$(BLACK)) $(4) $(RESET); \
elif [[ -n '$(3)' ]]; \
then printf '%b%b %s %b%b %s %b%b %s %b\n' $(call BG,$(DARK_GREY)) $(call FG,$(COLOR)) $(1) $(call BG,$(COLOR)) $(call FGB,$(BLACK)) $(2) $(call BG,$(DARK_GREY)) $(call FG,$(COLOR)) $(3) $(RESET); \
elif [[ -n '$(2)' ]]; \
then printf '%b%b %s %b%b %s %b\n' $(call BG,$(DARK_GREY)) $(call FGB,$(COLOR)) $(1) $(call BG,$(COLOR)) $(call FGB,$(BLACK)) $(2) $(RESET); \
elif [[ -n '$(1)' ]]; \
then printf '%b%b %s %b\n' $(call BG,$(COLOR)) $(call FGB,$(BLACK)) $(1) $(RESET); \
fi
endef
define print_h2
@printf '%b%b %s %b\n' $(call BG,$(LIGHT_GREY)) $(call FG,$(BLACK)) $(1) $(RESET)
endef
define print_h3
@printf '%b %s %b\n' $(call BG,$(DARK_GREY)) $(1) $(RESET)
endef
define print
@$(eval COLOR=$($(MAKE_THEME)))
@printf '%b %s %b\n' $(call FG,$(COLOR)) $(1) $(RESET)
endef
define print_space
@echo ""
endef
define print_options
@$(eval COLOR=$($(MAKE_THEME)))
@printf '%b %-15s %b%s%b\n' $(call FG,$(COLOR)) $(1) $(call FG,$(LIGHTER_GREY)) $(2) $(RESET)
endef
define print_warning
@printf '%b %s %b\n' $(call FGB,$(GOLD)) $(1) $(RESET)
endef
[tool.black]
line-length = 119
exclude = '''
^/(
blah/blah
| blah2/blah2
)
'''
\ No newline at end of file
# General packages
environs~=4.1.0
retry-decorator==1.1.1
# QA
flake8==4.0.1
pydocstyle==3.0.0
pep8-naming==0.13.1
flake8-debugger==4.1.2
flake8-print==5.0.0
flake8-todo==0.7
isort<5.0.0
flake8-isort==4.2.0
flake8-isort==4.2.0
black~=22.3.0
# Testing
pytest>=7.2.0
pytest-cov
pytest-mock
mock~=5.0.1
Faker==17.5.0
\ No newline at end of file
[flake8]
doctests = True
statistics = True
benchmark = True
max-line-length = 119
ignore = E203,D202,E501,W503
[tool:isort]
profile = "black"
combine_as_imports = true
multi_line_output=3
known_third_party=
environs,
include_trailing_comma=True
not_skip=__init__.py
line_length=119
[tool:pytest]
; Always run with long traceback and local variables on fail
addopts = --showlocals -p no:warnings
; ignore all files an folders that are hidden. This improves collection performance.
norecursedirs = .* tmp* ci src
# testpaths = tests
python_files = tests.py test_*.py *_tests.py
python_functions = test_*
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment