Compare commits
No commits in common. "db9fc35715eafa2977a0bc1ac0f78c28b092f30a" and "8e7d963e262506d057d62a34548f501cd1da75bc" have entirely different histories.
db9fc35715
...
8e7d963e26
|
|
@ -5,5 +5,3 @@
|
|||
/data
|
||||
backend/api_logs.txt
|
||||
*__pycache__*
|
||||
.env
|
||||
poetry.lock
|
||||
|
|
|
|||
|
|
@ -1,215 +0,0 @@
|
|||
def parse_int(args):
|
||||
try:
|
||||
return int( args )
|
||||
except ValueError:
|
||||
print(f"{args} cannot be cast to int")
|
||||
raise
|
||||
|
||||
def mangrove_by_country_latest():
|
||||
return """
|
||||
select * from models_final.final__protected_mangroves_summary_stats_by_country_agg
|
||||
where year = '2020'
|
||||
order by cumulative_pixels_diff desc
|
||||
"""
|
||||
|
||||
def bitcoin_business_growth_timeseries(args):
|
||||
days_ago = parse_int(args["days_ago"])
|
||||
country_name = args["country_name"]
|
||||
return f"""
|
||||
select * from models_final.final__bitcoin_business_growth_by_country
|
||||
where days_ago <= {days_ago} and country_name = '{country_name}'
|
||||
order by date
|
||||
"""
|
||||
|
||||
def bitcoin_business_growth_percent_diff_days_ago(args):
|
||||
days_ago = parse_int(args["days_ago"])
|
||||
return f"""
|
||||
with
|
||||
filtered_data as (
|
||||
select country_name, date, days_ago, cumulative_value
|
||||
from models_final.final__bitcoin_business_growth_by_country
|
||||
where days_ago <= {days_ago}
|
||||
order by country_name, days_ago desc
|
||||
),
|
||||
first_and_last_values as (
|
||||
select
|
||||
country_name,
|
||||
date,
|
||||
days_ago,
|
||||
cumulative_value,
|
||||
first_value(cumulative_value) over (
|
||||
partition by country_name order by days_ago desc
|
||||
) as first_value,
|
||||
first_value(date) over (
|
||||
partition by country_name order by days_ago desc
|
||||
) as first_date,
|
||||
first_value(cumulative_value) over (
|
||||
partition by country_name order by days_ago
|
||||
) as last_value,
|
||||
first_value(date) over (
|
||||
partition by country_name order by days_ago
|
||||
) as last_date
|
||||
from filtered_data
|
||||
),
|
||||
diff as (
|
||||
select
|
||||
country_name,
|
||||
date,
|
||||
first_date,
|
||||
last_date,
|
||||
days_ago,
|
||||
cumulative_value,
|
||||
first_value,
|
||||
last_value,
|
||||
last_value - first_value as difference,
|
||||
round(
|
||||
100 * safe_divide((last_value - first_value), first_value), 2
|
||||
) as percent_difference
|
||||
from first_and_last_values
|
||||
)
|
||||
select *
|
||||
from diff
|
||||
where days_ago = 1
|
||||
order by difference desc
|
||||
"""
|
||||
# def bitcoin_business_growth_timeseries(query):
|
||||
# pipeline = [
|
||||
# {
|
||||
# "$match": {
|
||||
# "days_ago": {"$lte": int(query["days_ago"])},
|
||||
# "country_name": query["country_name"],
|
||||
# }
|
||||
# },
|
||||
# {
|
||||
# "$project": {
|
||||
# "country_name": "$country_name",
|
||||
# "date": "$date",
|
||||
# "cumulative_value": "$cumulative_value",
|
||||
# }
|
||||
# },
|
||||
# {"$sort": {"country_name": 1, "days_ago": 1}},
|
||||
# ]
|
||||
# return pipeline
|
||||
# def mangrove_by_country_latest():
|
||||
# pipeline = [
|
||||
# {
|
||||
# "$match": {"year": "2020"},
|
||||
# },
|
||||
# ]
|
||||
# return pipeline
|
||||
#
|
||||
#
|
||||
# def mangrove_by_country_agg(query):
|
||||
# pipeline = [
|
||||
# {"$match": {"country_with_parent": query["country_with_parent"]}},
|
||||
# {
|
||||
# "$group": {
|
||||
# "_id": {"country_with_parent": "$country_with_parent", "year": "$year"},
|
||||
# "total_pixels": {"$sum": "$total_n_pixels"},
|
||||
# }
|
||||
# },
|
||||
# {
|
||||
# "$project": {
|
||||
# "_id": 0,
|
||||
# "country_with_parent": "$_id.country_with_parent",
|
||||
# "year": "$_id.year",
|
||||
# "total_pixels": 1,
|
||||
# }
|
||||
# },
|
||||
# {"$sort": {"year": 1}},
|
||||
# ]
|
||||
# return pipeline
|
||||
#
|
||||
#
|
||||
# def bitcoin_business_growth_timeseries(query):
|
||||
# pipeline = [
|
||||
# {
|
||||
# "$match": {
|
||||
# "days_ago": {"$lte": int(query["days_ago"])},
|
||||
# "country_name": query["country_name"],
|
||||
# }
|
||||
# },
|
||||
# {
|
||||
# "$project": {
|
||||
# "country_name": "$country_name",
|
||||
# "date": "$date",
|
||||
# "cumulative_value": "$cumulative_value",
|
||||
# }
|
||||
# },
|
||||
# {"$sort": {"country_name": 1, "days_ago": 1}},
|
||||
# ]
|
||||
# return pipeline
|
||||
#
|
||||
#
|
||||
# def bitcoin_business_growth_percent_diff_days_ago(query):
|
||||
pipeline = [
|
||||
{"$match": {"days_ago": {"$lte": int(query["days_ago"])}}},
|
||||
{"$sort": {"country_name": 1, "days_ago": 1}},
|
||||
{
|
||||
"$group": {
|
||||
"_id": "$country_name",
|
||||
"firstvalue": {"$first": "$cumulative_value"},
|
||||
"lastvalue": {"$last": "$cumulative_value"},
|
||||
"firstdate": {"$min": "$date"},
|
||||
"lastdate": {"$max": "$date"},
|
||||
}
|
||||
},
|
||||
{
|
||||
"$project": {
|
||||
"country_name": "$_id",
|
||||
"first_value": "$firstvalue",
|
||||
"last_value": "$lastvalue",
|
||||
"difference": {
|
||||
"$subtract": [
|
||||
{"$todouble": "$firstvalue"},
|
||||
{"$todouble": "$lastvalue"},
|
||||
]
|
||||
},
|
||||
"first_date": "$firstdate",
|
||||
"last_date": "$lastdate",
|
||||
"percent_difference": {
|
||||
"$cond": {
|
||||
"if": {"$eq": [{"$todouble": "$lastvalue"}, 0]},
|
||||
"then": {
|
||||
"$cond": {
|
||||
"if": {"$gt": [{"$todouble": "$firstvalue"}, 0]},
|
||||
"then": "new",
|
||||
"else": "none",
|
||||
}
|
||||
},
|
||||
"else": {
|
||||
"$round": [
|
||||
{
|
||||
"$multiply": [
|
||||
{
|
||||
"$divide": [
|
||||
{
|
||||
"$subtract": [
|
||||
{"$todouble": "$firstvalue"},
|
||||
{"$todouble": "$lastvalue"},
|
||||
]
|
||||
},
|
||||
{"$todouble": "$lastvalue"},
|
||||
]
|
||||
},
|
||||
100,
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
]
|
||||
return pipeline
|
||||
#
|
||||
#
|
||||
# def bitcoin_business_growth_latest(query):
|
||||
# pipeline = [
|
||||
# {
|
||||
# "$match": query["filter"],
|
||||
# },
|
||||
# {"$sort": {"date": 1}},
|
||||
# ]
|
||||
# return pipeline
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
from psycopg2.extras import RealDictCursor
|
||||
import psycopg2, os
|
||||
|
||||
class PostgresHandler:
|
||||
def __init__(self):
|
||||
self.connection = self.connect_to_pg()
|
||||
self.cur = self.connection.cursor(cursor_factory=RealDictCursor)
|
||||
|
||||
def connect_to_pg(self):
|
||||
try:
|
||||
connection = psycopg2.connect(
|
||||
dbname=os.getenv('PGDATABASE'),
|
||||
host=os.getenv('PGHOST'),
|
||||
user=os.getenv('PGUSER'),
|
||||
password=os.getenv('PGPASSWORD'),
|
||||
port=os.getenv('PGPORT'),
|
||||
)
|
||||
except Exception as e:
|
||||
message=f"Connection to postgres database failed: {e}"
|
||||
raise Exception(message)
|
||||
print(f"Successfully connected to DB")
|
||||
return connection
|
||||
|
||||
def execute_query(self, query):
|
||||
try:
|
||||
self.cur.execute(query)
|
||||
results = self.cur.fetchall()
|
||||
self.connection.commit()
|
||||
self.connection.close()
|
||||
return results
|
||||
except Exception:
|
||||
print("Error executing query")
|
||||
raise
|
||||
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
from fastapi import APIRouter
|
||||
from api.postgres_handler import PostgresHandler
|
||||
import api.pipelines as pipelines
|
||||
import api.schemas as schemas
|
||||
from api.schemas import DataSerializer
|
||||
import json
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
def parse_args_to_dict(query):
|
||||
try:
|
||||
return json.loads(query)
|
||||
except json.JSONDecodeError as e:
|
||||
return {"error": f"Invalid JSON: {e}"}
|
||||
|
||||
@router.get("/mangrove_by_country_latest")
|
||||
async def mangrove_by_country_latest():
|
||||
pipeline = pipelines.mangrove_by_country_latest()
|
||||
handler = PostgresHandler()
|
||||
|
||||
schema = schemas.mangrove_by_country_latest_schema
|
||||
serializer = DataSerializer(schema)
|
||||
rawData = handler.execute_query(pipeline)
|
||||
serializedData = serializer.serialize_many(rawData)
|
||||
return serializedData
|
||||
|
||||
@router.get("/bitcoin_business_growth_timeseries")
|
||||
async def bitcoin_business_growth_timeseries(query: str):
|
||||
args = parse_args_to_dict(query)
|
||||
|
||||
pipeline = pipelines.bitcoin_business_growth_timeseries(args)
|
||||
handler = PostgresHandler()
|
||||
|
||||
schema = schemas.bitcoin_business_growth_timeseries_schema
|
||||
serializer = DataSerializer(schema)
|
||||
|
||||
rawData = handler.execute_query(pipeline)
|
||||
serializedData = serializer.serialize_many(rawData)
|
||||
|
||||
return serializedData
|
||||
|
||||
@router.get("/bitcoin_business_growth_percent_diff")
|
||||
async def bitcoin_business_growth_percent_diff(query: str):
|
||||
args = parse_args_to_dict(query)
|
||||
|
||||
pipeline = pipelines.bitcoin_business_growth_percent_diff_days_ago(args)
|
||||
handler = PostgresHandler()
|
||||
|
||||
schema = schemas.bitcoin_business_growth_percent_diff_schema
|
||||
serializer = DataSerializer(schema)
|
||||
|
||||
rawData = handler.execute_query(pipeline)
|
||||
serializedData = serializer.serialize_many(rawData)
|
||||
return serializedData
|
||||
|
||||
# @router.get("/bitcoin_business_growth_percent_diff")
|
||||
# async def bitcoin_business_growth_percent_diff(query: str):
|
||||
# query = ast.literal_eval(query)
|
||||
#
|
||||
# query = queries.bitcoin_business_growth_percent_diff_days_ago(query)
|
||||
# handler = PostgresHandler(connection)
|
||||
#
|
||||
# schema = schemas.bitcoin_business_growth_percent_diff_schema
|
||||
# pipeline = pipelines.bitcoin_business_growth_percent_diff_days_ago(query)
|
||||
# serializer = DataSerializer(schema)
|
||||
# handler = MongoDBHandler(collection_name)
|
||||
# rawData = handler.aggregate(pipeline)
|
||||
# serializedData = serializer.serialize_many(rawData)
|
||||
# return serializedData
|
||||
# @router.get("/mangrove_by_country_agg")
|
||||
# async def mangrove_by_country_agg(query: str):
|
||||
# query = ast.literal_eval(query)
|
||||
# db = client.baseddata
|
||||
# collection_name = db["final__protected_mangroves_summary_stats_by_country_agg"]
|
||||
# schema = schemas.mangrove_by_country_agg_schema
|
||||
# pipeline = pipelines.mangrove_by_country_agg(query)
|
||||
# serializer = DataSerializer(schema)
|
||||
# handler = MongoDBHandler(collection_name)
|
||||
# rawData = handler.aggregate(pipeline)
|
||||
# serializedData = serializer.serialize_many(rawData)
|
||||
# return serializedData
|
||||
#
|
||||
|
||||
# @router.get("/bitcoin_business_growth_timeseries")
|
||||
# async def bitcoin_business_growth_timeseries(query: str):
|
||||
# query = ast.literal_eval(query)
|
||||
# db = client.baseddata
|
||||
# collection_name = db["final__bitcoin_business_growth_by_country"]
|
||||
# schema = schemas.bitcoin_business_growth_timeseries_schema
|
||||
# pipeline = pipelines.bitcoin_business_growth_timeseries(query)
|
||||
# serializer = DataSerializer(schema)
|
||||
# handler = MongoDBHandler(collection_name)
|
||||
# rawData = handler.aggregate(pipeline)
|
||||
# serializedData = serializer.serialize_many(rawData)
|
||||
# return serializedData
|
||||
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
def mangrove_by_country_latest_schema(data):
|
||||
return {
|
||||
"country_with_parent": str(data["country_with_parent"]),
|
||||
"original_pixels": int(data["original_pixels"]),
|
||||
"total_n_pixels": int(data["total_n_pixels"]),
|
||||
"cumulative_pixels_diff": int(data["cumulative_pixels_diff"]),
|
||||
"cumulative_pct_diff": float(data["cumulative_pct_diff"]),
|
||||
}
|
||||
|
||||
def mangrove_by_country_agg_schema(data):
|
||||
return {
|
||||
"country_with_parent": str(data["country_with_parent"]),
|
||||
"year": int(data["year"]),
|
||||
"total_pixels": int(data["total_pixels"])
|
||||
}
|
||||
|
||||
def bitcoin_business_growth_percent_diff_schema(data):
|
||||
return {
|
||||
"country_name": str(data["country_name"]),
|
||||
"date_range": str(f'{data["first_date"]} to {data["last_date"]}'),
|
||||
"first_value": int(data["first_value"]),
|
||||
"last_value": int(data["last_value"]),
|
||||
"difference": int(data["difference"]),
|
||||
"percent_difference": str(data["percent_difference"])
|
||||
}
|
||||
|
||||
def bitcoin_business_growth_timeseries_schema(data):
|
||||
return {
|
||||
"country_name": str(data["country_name"]),
|
||||
"date": data["date"],
|
||||
"cumulative_value": int(data["cumulative_value"])
|
||||
}
|
||||
|
||||
class DataSerializer:
|
||||
def __init__(self, schema_func):
|
||||
self.schema_func = schema_func
|
||||
|
||||
def serialize_one(self, data) -> dict:
|
||||
return self.schema_func(dict( data ))
|
||||
|
||||
def serialize_many(self, data_list) -> list:
|
||||
return [self.serialize_one(data) for data in data_list]
|
||||
|
|
@ -0,0 +1,210 @@
|
|||
from flask import Flask, g, jsonify, request, json, Response, send_from_directory, abort
|
||||
from flask_cors import CORS
|
||||
import orjson, os
|
||||
import pandas as pd
|
||||
|
||||
import datetime
|
||||
import time
|
||||
|
||||
app = Flask(__name__)
|
||||
CORS(app)
|
||||
|
||||
FILES_DIRECTORY = "../data/"
|
||||
|
||||
|
||||
@app.before_request
|
||||
def start_timer():
|
||||
g.start = time.time()
|
||||
|
||||
|
||||
@app.after_request
|
||||
def log(response):
|
||||
now = time.time()
|
||||
duration = round(now - g.start, 4)
|
||||
dt = datetime.datetime.fromtimestamp(now).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
log_entry = {
|
||||
"timestamp": dt,
|
||||
"duration": duration,
|
||||
"method": request.method,
|
||||
"url": request.url,
|
||||
"status": response.status_code,
|
||||
"remote_addr": request.access_route[-1],
|
||||
"user_agent": request.user_agent.string,
|
||||
}
|
||||
|
||||
log_line = ",".join(f"{key}={value}" for key, value in log_entry.items())
|
||||
|
||||
with open("api_logs.txt", "a") as f:
|
||||
f.write(log_line + "\n")
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@app.route("/bitcoin_business_growth_by_country", methods=["GET"])
|
||||
def business_growth():
|
||||
|
||||
today = datetime.datetime.today()
|
||||
|
||||
# Parse args from request
|
||||
latest_date = request.args.get("latest_date")
|
||||
country_names = request.args.get("countries")
|
||||
cumulative_period_type = request.args.get("cumulative_period_type")
|
||||
|
||||
# Open json locally
|
||||
with open("../data/final__bitcoin_business_growth_by_country.json", "rb") as f:
|
||||
data = orjson.loads(f.read())
|
||||
|
||||
# Filter based on args
|
||||
if latest_date:
|
||||
latest_date_bool = latest_date == "true"
|
||||
filtered_data = [
|
||||
item for item in data if item["latest_date"] == latest_date_bool
|
||||
]
|
||||
else:
|
||||
filtered_data = data
|
||||
|
||||
if country_names:
|
||||
countries = [name.strip() for name in country_names.split(",")]
|
||||
filtered_data = [
|
||||
item for item in filtered_data if item["country_name"] in countries
|
||||
]
|
||||
|
||||
if cumulative_period_type == "1 day":
|
||||
delta = today - datetime.timedelta(days=2)
|
||||
filtered_data = [
|
||||
item
|
||||
for item in filtered_data
|
||||
if item["cumulative_period_type"] == cumulative_period_type
|
||||
and delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d")
|
||||
]
|
||||
elif cumulative_period_type == "7 day":
|
||||
delta = today - datetime.timedelta(days=8)
|
||||
filtered_data = [
|
||||
item
|
||||
for item in filtered_data
|
||||
if item["cumulative_period_type"] == cumulative_period_type
|
||||
and delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d")
|
||||
]
|
||||
elif cumulative_period_type == "28 day":
|
||||
delta = today - datetime.timedelta(days=29)
|
||||
filtered_data = [
|
||||
item
|
||||
for item in filtered_data
|
||||
if item["cumulative_period_type"] == cumulative_period_type
|
||||
and delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d")
|
||||
]
|
||||
elif cumulative_period_type == "365 day":
|
||||
delta = today - datetime.timedelta(days=366)
|
||||
filtered_data = [
|
||||
item
|
||||
for item in filtered_data
|
||||
if item["cumulative_period_type"] == cumulative_period_type
|
||||
and delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d")
|
||||
]
|
||||
|
||||
# Sort by date
|
||||
sorted_data = sorted(filtered_data, key=lambda x: x["date"], reverse=False)
|
||||
|
||||
# Return json
|
||||
return Response(json.dumps(sorted_data), mimetype="application/json")
|
||||
|
||||
|
||||
@app.route("/get_json/<filename>", methods=["GET"])
|
||||
def get_json(filename):
|
||||
|
||||
period = request.args.get("period")
|
||||
today = datetime.datetime.today()
|
||||
|
||||
file_path = os.path.join(FILES_DIRECTORY, filename)
|
||||
if not os.path.isfile(file_path):
|
||||
abort(404)
|
||||
|
||||
with open(file_path, "r") as f:
|
||||
data = orjson.loads(f.read())
|
||||
|
||||
if period == "last 7 days":
|
||||
delta = today - datetime.timedelta(days=7)
|
||||
filtered_data = [
|
||||
item
|
||||
for item in data
|
||||
if delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d") <= today
|
||||
]
|
||||
sorted_data = sorted(filtered_data, key=lambda x: x["date"])
|
||||
elif period == "last 28 days":
|
||||
delta = today - datetime.timedelta(days=28)
|
||||
filtered_data = [
|
||||
item
|
||||
for item in data
|
||||
if delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d") <= today
|
||||
]
|
||||
sorted_data = sorted(filtered_data, key=lambda x: x["date"])
|
||||
elif period == "last 365 days":
|
||||
delta = today - datetime.timedelta(days=365)
|
||||
filtered_data = [
|
||||
item
|
||||
for item in data
|
||||
if delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d") <= today
|
||||
]
|
||||
sorted_data = sorted(filtered_data, key=lambda x: x["date"])
|
||||
elif period == "last 2 years":
|
||||
delta = today - datetime.timedelta(days=730)
|
||||
filtered_data = [
|
||||
item
|
||||
for item in data
|
||||
if delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d") <= today
|
||||
]
|
||||
sorted_data = sorted(filtered_data, key=lambda x: x["date"])
|
||||
else:
|
||||
sorted_data = sorted(data, key=lambda x: x["date"])
|
||||
|
||||
return jsonify(sorted_data)
|
||||
|
||||
|
||||
@app.route("/mangrove_data/<method>", methods=["GET"])
|
||||
def mangrove_data(method):
|
||||
|
||||
with open("../data/dev/final__wdpa_pid_mangrove_diff_stats.json", "rb") as f:
|
||||
data = orjson.loads(f.read())
|
||||
|
||||
if method == "countries":
|
||||
df = pd.read_json(json.dumps(data))
|
||||
countries = df[["year", "country", "n_pixels", "diff", "cumulative_diff"]]
|
||||
countriesAgg = countries.groupby(["year", "country"]).agg(
|
||||
{"n_pixels": "sum", "diff": "sum", "cumulative_diff": "sum"}
|
||||
)
|
||||
countriesAgg["year0_pixels"] = (
|
||||
countriesAgg["n_pixels"] - countriesAgg["cumulative_diff"]
|
||||
)
|
||||
countriesAgg["pct_diff"] = (
|
||||
100
|
||||
* (countriesAgg["n_pixels"] - countriesAgg["year0_pixels"])
|
||||
/ countriesAgg["year0_pixels"]
|
||||
).round(2)
|
||||
countriesLatest = countriesAgg.loc[[2020]].reset_index().set_index("country")
|
||||
return Response(
|
||||
countriesLatest.to_json(orient="index"), mimetype="application/json"
|
||||
)
|
||||
|
||||
|
||||
@app.route("/download/<filename>", methods=["GET"])
|
||||
def download_file(filename):
|
||||
try:
|
||||
return send_from_directory(FILES_DIRECTORY, filename, as_attachment=True)
|
||||
except FileNotFoundError:
|
||||
abort(404)
|
||||
|
||||
|
||||
@app.route("/cog", methods=["GET"])
|
||||
def serve_cog():
|
||||
year = request.args.get("year")
|
||||
pid = request.args.get("pid") # change this line
|
||||
dir = f"{FILES_DIRECTORY}/cog/{year}/"
|
||||
try:
|
||||
return send_from_directory(dir, f"{pid}.tif", as_attachment=True)
|
||||
except FileNotFoundError:
|
||||
abort(404)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run()
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
from fastapi import FastAPI
|
||||
from api.route import router
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
app.include_router(router)
|
||||
|
|
@ -15,10 +15,8 @@ You can select the growth period of interest from the drop-down, which updates t
|
|||
The chart always reflects the countries selected in the table.
|
||||
|
||||
<br/>
|
||||
{{< dropdown_filter id="days_ago_dropdown_filter" id_filter="days_ago" options="1 day:1,7 day:7,28 day:28,1 year:365,5 year:1826,10 year:3652,all time:10000" default_selection="7 day" targets="bitcoin-business-growth-chart bitcoin-business-growth-table" >}}
|
||||
{{< table id="bitcoin-business-growth-table" endpoint="bitcoin_business_growth_percent_diff" headers="{'country_name': 'Country', 'date_range': 'Date Range', 'first_value': 'Previous #', 'last_value': 'Current #', 'difference': 'Diff', 'percent_difference': '% Diff'}" maxHeight="400px" sortable="true" valueId="country_name" selectableRows="multi" targets="bitcoin-business-growth-chart" defaultFirstSelected="true" >}}
|
||||
|
||||
{{< chart id="bitcoin-business-growth-chart" endpoint="bitcoin_business_growth_timeseries" chartType="line" xAxisField="date" yAxisField="cumulative_value" scaleChart=true >}}
|
||||
{{< chart src="/js/bitcoin-business-growth-chart.js" >}}
|
||||
{{< table src="/js/bitcoin-business-growth-table.js" >}}
|
||||
|
||||
#### Attribution and License
|
||||
Data obtained from © [OpenStreetMap](https://www.openstreetmap.org/copyright)
|
||||
|
|
|
|||
|
|
@ -9,8 +9,5 @@ tags: ["Bitcoin", "Stats"]
|
|||
script: "/js/mangrove-map.js"
|
||||
---
|
||||
|
||||
{{< table id="mangrove_countries" endpoint="mangrove_by_country_latest" headers="{'country_with_parent': 'Country', 'original_pixels': '1996 Cover', 'total_n_pixels': '2020 Cover', 'cumulative_pixels_diff': 'Diff', 'cumulative_pct_diff': '% Diff'}" maxHeight="400px" sortable="true" valueId="country_with_parent" selectableRows="single" defaultFirstSelected="true" >}}
|
||||
{{< chart id="mangrove_countries" endpoint="mangrove_by_country_agg" chartType="bar" xAxisField="year" yAxisField="total_pixels" scaleChart=true >}}
|
||||
{{< table id="mangrove_countries" endpoint="mangrove_data/countries" tableKey="country" headers="{'country': 'Country', 'cumulative_diff': 'Diff', 'pct_diff': '% Diff'}" maxHeight="400px" sortable="true">}}
|
||||
{{< map id="map" style="https://tiles.semitamaps.com/styles/maptiler-basic/style.json">}}
|
||||
|
||||
{{< chart id="mangrove-country-timeseries-chart" endpoint="mangrove_country_timeseries" chartType="line" xAxisField="date" yAxisField="n_pixels" scaleChart=true >}}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ languageCode = 'en-gb'
|
|||
title = 'Based Data'
|
||||
|
||||
[params]
|
||||
apiURL = 'http://localhost:8000'
|
||||
apiURL = 'http://localhost:5000'
|
||||
|
||||
[markup.highlight]
|
||||
pygmentsUseClasses = false
|
||||
|
|
|
|||
|
|
@ -1,96 +1,8 @@
|
|||
<script src="https://cdn.jsdelivr.net/npm/echarts@5.5.1/dist/echarts.min.js"></script>
|
||||
<script>
|
||||
chartData = [];
|
||||
function createChart(
|
||||
id,
|
||||
endpoint,
|
||||
chartType,
|
||||
xAxisField,
|
||||
yAxisField,
|
||||
sortField = null,
|
||||
scaleChart = false,
|
||||
) {
|
||||
async function fetchDataForChart(query, valueId) {
|
||||
try {
|
||||
const apiEndpoint = `${apiURL}/${endpoint}?${query}`;
|
||||
const response = await fetch(apiEndpoint);
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
const fetchedData = await response.json();
|
||||
const newData = fetchedData.reduce((acc, item) => {
|
||||
const objectId = item[valueId];
|
||||
if (!acc[objectId]) {
|
||||
acc[objectId] = [];
|
||||
}
|
||||
acc[objectId].push([item[xAxisField], item[yAxisField]]);
|
||||
return acc;
|
||||
}, {});
|
||||
chartData = { ...chartData, ...newData };
|
||||
updateChart();
|
||||
} catch (error) {
|
||||
console.error("Fetching data failed:", error);
|
||||
}
|
||||
}
|
||||
|
||||
function updateChart() {
|
||||
console.log(chartData);
|
||||
let chartDataMap = new Map();
|
||||
for (let objectId in chartData) {
|
||||
chartDataMap.set(objectId, chartData[objectId]);
|
||||
}
|
||||
var chartDom = document.getElementById(`${id}`);
|
||||
var myChart = echarts.init(chartDom);
|
||||
|
||||
var option = {
|
||||
tooltip: {
|
||||
...tooltip,
|
||||
valueFormatter(value, index) {
|
||||
return nFormatter(value, 0);
|
||||
},
|
||||
},
|
||||
xAxis: {
|
||||
type: "time",
|
||||
},
|
||||
yAxis: {
|
||||
scale: scaleChart,
|
||||
type: "value",
|
||||
},
|
||||
series: Array.from(chartDataMap.entries()).map(([name, data]) => ({
|
||||
name,
|
||||
type: chartType,
|
||||
data,
|
||||
showSymbol: false,
|
||||
})),
|
||||
};
|
||||
|
||||
myChart.setOption(option, true);
|
||||
}
|
||||
|
||||
// listen for filter events for this target
|
||||
document.addEventListener("filterChange", function (event) {
|
||||
tableId = document.getElementById(id).id;
|
||||
console.log(event.detail);
|
||||
eventDetail = event.detail;
|
||||
if (eventDetail.filterActions.includes("refresh")) {
|
||||
chartData = [];
|
||||
updateChart();
|
||||
} else {
|
||||
if (eventDetail.filterTargets.includes(tableId)) {
|
||||
if (eventDetail.filterActions.includes("selected")) {
|
||||
valueId = eventDetail.filterId;
|
||||
let selectedRow = {
|
||||
[valueId]: eventDetail.filterValue,
|
||||
};
|
||||
query = queryConstructor(selectedRow);
|
||||
fetchDataForChart(query, valueId);
|
||||
} else {
|
||||
delete chartData[eventDetail.filterValue];
|
||||
updateChart();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
</script>
|
||||
<section class="chart-container">
|
||||
<div id="chart-modifiers"></div>
|
||||
<div id="chart">
|
||||
<script src="{{ .src }}"></script>
|
||||
</div>
|
||||
</section>
|
||||
<script src="/js/chart-params.js"></script>
|
||||
|
|
|
|||
|
|
@ -8,8 +8,6 @@
|
|||
<link rel="stylesheet" href="/css/toc.css" type="text/css" media="all" />
|
||||
<link rel="stylesheet" href="/css/articles.css" type="text/css" media="all" />
|
||||
<link rel="stylesheet" href="/css/charts.css" type="text/css" media="all" />
|
||||
<script src="/js/lib/sorttable.js"></script>
|
||||
<script src="/js/lib/helper-functions.js"></script>
|
||||
<link
|
||||
rel="stylesheet"
|
||||
href="/css/codeblock.css"
|
||||
|
|
|
|||
|
|
@ -1,142 +0,0 @@
|
|||
<script>
|
||||
function createTable(
|
||||
endpoint,
|
||||
id,
|
||||
headers,
|
||||
maxHeight,
|
||||
sortable,
|
||||
valueId,
|
||||
selectableRows,
|
||||
filterTargets,
|
||||
defaultFirstSelected,
|
||||
) {
|
||||
async function fetchDataForTable(query) {
|
||||
try {
|
||||
const apiEndpoint = `${apiURL}/${endpoint}?${query}`;
|
||||
const response = await fetch(apiEndpoint);
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
const fetchedData = await response.json();
|
||||
data = fetchedData;
|
||||
generateTable(data);
|
||||
} catch (error) {
|
||||
console.error("Fetching data failed:", error);
|
||||
}
|
||||
}
|
||||
|
||||
function generateTable(data) {
|
||||
const jsonTableContainer = document.getElementById(`${id}--container`);
|
||||
jsonTableContainer.className = "jsonTableContainer";
|
||||
jsonTableContainer.innerHTML = "";
|
||||
jsonTableContainer.style.maxHeight = maxHeight;
|
||||
|
||||
tableHeaderNames = Object.values(headers);
|
||||
tableHeaderKeys = Object.keys(headers);
|
||||
|
||||
const table = document.createElement("table");
|
||||
table.id = `${id}`;
|
||||
const thead = document.createElement("thead");
|
||||
const tbody = document.createElement("tbody");
|
||||
const headerRow = document.createElement("tr");
|
||||
tableHeaderNames.forEach((header) => {
|
||||
const th = document.createElement("th");
|
||||
th.textContent = header;
|
||||
headerRow.appendChild(th);
|
||||
});
|
||||
thead.appendChild(headerRow);
|
||||
table.appendChild(thead);
|
||||
|
||||
for (const key in data) {
|
||||
const row = document.createElement("tr");
|
||||
row.value = data[key][valueId];
|
||||
tableHeaderKeys.forEach((columnName) => {
|
||||
const td = document.createElement("td");
|
||||
const div = document.createElement("div");
|
||||
div.id = "scrollable";
|
||||
div.textContent = data[key][columnName];
|
||||
td.appendChild(div);
|
||||
row.appendChild(td);
|
||||
tbody.appendChild(row);
|
||||
});
|
||||
}
|
||||
table.appendChild(thead);
|
||||
table.appendChild(tbody);
|
||||
jsonTableContainer.appendChild(table);
|
||||
|
||||
// sortable
|
||||
if (sortable == "true") {
|
||||
table.className = "sortable";
|
||||
sorttable.makeSortable(document.getElementById(`${id}`));
|
||||
}
|
||||
|
||||
if (selectableRows === "multi" || selectableRows === "single") {
|
||||
const rows = table.getElementsByTagName("tr");
|
||||
for (let i = 1; i < rows.length; i++) {
|
||||
rows[i].addEventListener("click", function () {
|
||||
if (selectableRows === "multi") {
|
||||
this.classList.toggle("selected");
|
||||
if (this.classList.contains("selected")) {
|
||||
const event = new CustomEvent("filterChange", {
|
||||
detail: {
|
||||
filterId: valueId,
|
||||
filterValue: this.value,
|
||||
filterActions: ["selected"],
|
||||
filterTargets: filterTargets,
|
||||
},
|
||||
});
|
||||
document.dispatchEvent(event);
|
||||
} else {
|
||||
const event = new CustomEvent("filterChange", {
|
||||
detail: {
|
||||
filterId: valueId,
|
||||
filterValue: this.value,
|
||||
filterActions: ["deselected"],
|
||||
filterTargets: filterTargets,
|
||||
},
|
||||
});
|
||||
document.dispatchEvent(event);
|
||||
}
|
||||
} else if (selectableRows === "single") {
|
||||
if (this.classList.contains("selected")) {
|
||||
this.classList.remove("selected");
|
||||
} else {
|
||||
for (let j = 1; j < rows.length; j++) {
|
||||
rows[j].classList.remove("selected");
|
||||
}
|
||||
this.classList.add("selected");
|
||||
}
|
||||
}
|
||||
});
|
||||
if (defaultFirstSelected == true) {
|
||||
if (i == 1) {
|
||||
rows[i].classList.add("selected");
|
||||
const event = new CustomEvent("filterChange", {
|
||||
detail: {
|
||||
filterId: valueId,
|
||||
filterValue: rows[i].value,
|
||||
filterActions: ["selected"],
|
||||
filterTargets: filterTargets,
|
||||
},
|
||||
});
|
||||
document.dispatchEvent(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// listen for filter events for this target
|
||||
document.addEventListener("filterChange", function (event) {
|
||||
tableId = document.getElementById(id).id;
|
||||
if (event.detail.filterTargets.includes(tableId)) {
|
||||
query = queryConstructor();
|
||||
fetchDataForTable(query);
|
||||
}
|
||||
});
|
||||
|
||||
query = queryConstructor();
|
||||
fetchDataForTable(query);
|
||||
}
|
||||
</script>
|
||||
|
|
@ -1,10 +1,2 @@
|
|||
{{ partial "chart.html" }}
|
||||
<section class = 'chart-container'>
|
||||
<div class = "chart" id='{{ .Get "id" }}'>
|
||||
<script>
|
||||
document.addEventListener("DOMContentLoaded", function () {
|
||||
createChart(id={{ .Get "id" }}, endpoint={{ .Get "endpoint" }}, chartType={{ .Get "chartType" }}, xAxisField={{ .Get "xAxisField" }}, yAxisField={{ .Get "yAxisField" }}, sortField={{ .Get "sortField" }}, scaleChart={{ .Get "scaleChart" }})
|
||||
});
|
||||
</script>
|
||||
</div>
|
||||
</section>
|
||||
{{ $id := .Get "src" | md5 }} {{ partial "chart.html" (dict "src" (.Get "src")
|
||||
"id" $id) }}
|
||||
|
|
|
|||
|
|
@ -1,29 +0,0 @@
|
|||
{{ $id := .Get "id" }}
|
||||
{{ $default_selection := .Get "default_selection" }}
|
||||
{{ $options := .Get "options" }}
|
||||
|
||||
{{ $options_split := split $options "," }}
|
||||
|
||||
<div class="dropdown-filter-container">
|
||||
<select class="filter dropdown-filter" id="{{ $id }}" idFilter='{{ .Get "id_filter" }}' onchange="dispatchDropdownEvent(this)">
|
||||
{{ range $options_split }}
|
||||
{{ $parts := split . ":" }}
|
||||
{{ $key := index $parts 0 }}
|
||||
{{ $value := index $parts 1 }}
|
||||
<option value="{{ $value }}" {{ if eq $key $default_selection }}selected{{ end }}>{{ $key }}</option>
|
||||
{{ end }}
|
||||
</select>
|
||||
<script>
|
||||
function dispatchDropdownEvent(selectElement) {
|
||||
const event = new CustomEvent('filterChange', {
|
||||
detail: {
|
||||
filterId: '{{ .Get "id_filter" }}',
|
||||
filterValue: selectElement.value,
|
||||
filterActions: ["refresh"],
|
||||
filterTargets: '{{ .Get "targets" }}'.split(" ")
|
||||
}
|
||||
});
|
||||
document.dispatchEvent(event);
|
||||
}
|
||||
</script>
|
||||
</div>
|
||||
|
|
@ -1,8 +1,67 @@
|
|||
{{ partial "table.html" }}
|
||||
<div id = '{{ .Get "id" }}--container'>
|
||||
<script>
|
||||
document.addEventListener("DOMContentLoaded", function () {
|
||||
createTable({{ .Get "endpoint" }}, {{ .Get "id" }}, {{ .Get "headers" | safeJS }}, {{ .Get "maxHeight" }}, {{ .Get "sortable" }}, {{ .Get "valueId" }}, {{ .Get "selectableRows" }}, '{{ .Get "targets" }}'.split(" "), {{ .Get "defaultFirstSelected" | safeJS }})
|
||||
});
|
||||
async function fetchDataForTable() {
|
||||
try {
|
||||
const apiEndpoint = `${apiURL}/{{ .Get "endpoint" }}`;
|
||||
const response = await fetch(apiEndpoint);
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
const fetchedData = await response.json();
|
||||
data = fetchedData;
|
||||
console.log(data);
|
||||
createTable(data);
|
||||
} catch (error) {
|
||||
console.error("Fetching data failed:", error);
|
||||
}
|
||||
}
|
||||
|
||||
function createTable(data) {
|
||||
const jsonTableContainer = document.getElementById("jsonTableContainer");
|
||||
jsonTableContainer.innerHTML = "";
|
||||
jsonTableContainer.style.maxHeight = "{{ .Get `maxHeight` }}"
|
||||
|
||||
tableHeaderNames = Object.values({{ .Get `headers` | safeJS }});
|
||||
tableHeaderKeys = Object.keys({{ .Get `headers` | safeJS }});
|
||||
|
||||
const table = document.createElement("table");
|
||||
table.id = "{{ .Get `id` }}"
|
||||
const thead = document.createElement("thead");
|
||||
const tbody = document.createElement("tbody");
|
||||
const headerRow = document.createElement("tr");
|
||||
tableHeaderNames.forEach((header) => {
|
||||
const th = document.createElement("th");
|
||||
th.textContent = header;
|
||||
headerRow.appendChild(th);
|
||||
});
|
||||
thead.appendChild(headerRow);
|
||||
table.appendChild(thead);
|
||||
|
||||
for (const key in data) {
|
||||
const row = document.createElement('tr');
|
||||
tableHeaderKeys.forEach((columnName) => {
|
||||
const td = document.createElement("td");
|
||||
const div = document.createElement("div");
|
||||
div.id = "scrollable";
|
||||
if (columnName == "{{ .Get `tableKey` }}") {
|
||||
div.textContent = key;
|
||||
} else {
|
||||
div.textContent = data[key][columnName];
|
||||
};
|
||||
td.appendChild(div);
|
||||
row.appendChild(td);
|
||||
tbody.appendChild(row);
|
||||
});
|
||||
}
|
||||
table.appendChild(thead);
|
||||
table.appendChild(tbody);
|
||||
jsonTableContainer.appendChild(table)
|
||||
{{ if eq (.Get "sortable") "true" }}
|
||||
table.className = "sortable"
|
||||
sorttable.makeSortable(document.getElementById("{{ .Get `id` }}"));
|
||||
{{ end }}
|
||||
|
||||
table.className = ""
|
||||
}
|
||||
</script>
|
||||
</div>
|
||||
<div id="jsonTableContainer"></div>
|
||||
<script src="/js/lib/sorttable.js"></script>
|
||||
|
|
|
|||
|
|
@ -1,22 +0,0 @@
|
|||
[tool.poetry]
|
||||
name = "baseddata-io"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = ["Sam <samual.shop@proton.me>"]
|
||||
readme = "README.md"
|
||||
packages = [{include = "baseddata"}]
|
||||
package-mode = false
|
||||
|
||||
[virtualenvs]
|
||||
in-project = true
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.11"
|
||||
fastapi = "^0.115.4"
|
||||
uvicorn = "^0.32.0"
|
||||
psycopg2 = "^2.9.10"
|
||||
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
22
shell.nix
22
shell.nix
|
|
@ -1,19 +1,20 @@
|
|||
{pkgs ? import <nixpkgs> {}}:
|
||||
{ pkgs ? import <nixpkgs> { } }:
|
||||
|
||||
pkgs.mkShell
|
||||
{
|
||||
nativeBuildInputs = with pkgs; [
|
||||
python312Packages.flask
|
||||
python312Packages.flask-cors
|
||||
python312Packages.requests
|
||||
python312Packages.pandas
|
||||
python312Packages.orjson
|
||||
hugo
|
||||
python311
|
||||
poetry
|
||||
tmux
|
||||
];
|
||||
|
||||
NIX_LD_LIBRARY_PATH=/run/current-system/sw/share/nix-ld/lib;
|
||||
NIX_LD=/run/current-system/sw/share/nix-ld/lib/ld.so;
|
||||
shellHook = ''
|
||||
${pkgs.cowsay}/bin/cowsay "Welcome to the baseddata.io development environment!" | ${pkgs.lolcat}/bin/lolcat
|
||||
export LD_LIBRARY_PATH=$NIX_LD_LIBRARY_PATH
|
||||
source .env
|
||||
source .venv/bin/activate
|
||||
|
||||
get_session=$(tmux list-session | grep "baseddata")
|
||||
|
||||
if [ -z "$get_session" ];
|
||||
|
|
@ -21,8 +22,9 @@ pkgs.mkShell
|
|||
tmux new-session -d -s baseddata
|
||||
tmux split-window -h
|
||||
tmux send-keys -t 0 "hugo server" C-m
|
||||
tmux send-keys -t 1 "cd backend && uvicorn main:app --reload" C-m
|
||||
tmux send-keys -t 1 "cd backend && python app.py" C-m
|
||||
echo "Baseddata running in dev tmux shell"
|
||||
fi
|
||||
'';
|
||||
'';
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,24 @@
|
|||
/* Charts */
|
||||
.chart-container {
|
||||
.chart-flex-container {
|
||||
display: flex;
|
||||
/* height: 600px; */
|
||||
aspect-ratio: 1 / 1;
|
||||
}
|
||||
|
||||
.chart {
|
||||
display: flex;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
.chart-flex-container article {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
#chart {
|
||||
width: 100%;
|
||||
aspect-ratio: 16 / 9;
|
||||
}
|
||||
|
||||
.chart-container {
|
||||
margin-top: 20px;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
#chart-modifiers {
|
||||
display: flex;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,9 @@
|
|||
/* Tables */
|
||||
.jsonTableContainer {
|
||||
#jsonTableContainer {
|
||||
display: flex;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
table.sortable th:not(.sorttable_sorted):not(.sorttable_sorted_reverse):not(.sorttable_nosort):after {
|
||||
content: " \25B4\25BE"
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
|
|
@ -53,10 +49,3 @@ tr:nth-child(odd) {
|
|||
background-color: var(--table-odd-row-bg-color);
|
||||
font-size: var(--table-row-font-size);
|
||||
}
|
||||
tr:hover {
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
|
||||
tr.selected {
|
||||
background-color: #d1ecf1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,16 +0,0 @@
|
|||
function queryConstructor(customFilters = {}) {
|
||||
let filters = document.querySelectorAll(".filter");
|
||||
let queryObject = {};
|
||||
|
||||
Object.assign(queryObject, customFilters);
|
||||
|
||||
filters.forEach((filter) => {
|
||||
const filterId = filter.getAttribute("idFilter");
|
||||
const filterValue = filter.value;
|
||||
queryObject[filterId] = filterValue;
|
||||
});
|
||||
|
||||
let queryString = `query=${JSON.stringify(queryObject)}`;
|
||||
|
||||
return queryString;
|
||||
}
|
||||
|
|
@ -15,3 +15,6 @@ map.on("load", () => {
|
|||
});
|
||||
});
|
||||
|
||||
document.addEventListener("DOMContentLoaded", function () {
|
||||
fetchDataForTable();
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue