sam
/
baseddata.io
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: sam:db9fc35715eafa2977a0bc1ac0f78c28b092f30a
Branches Tags
sam:master
sam:dev
sam:new-backend
sam:data-downloads
..
compare: sam:8e7d963e262506d057d62a34548f501cd1da75bc
Branches Tags
sam:dev
sam:new-backend
sam:master
sam:data-downloads

No commits in common. "db9fc35715eafa2977a0bc1ac0f78c28b092f30a" and "8e7d963e262506d057d62a34548f501cd1da75bc" have entirely different histories.
db9fc35715 ... 8e7d963e26

22 changed files with 323 additions and 755 deletions
Show Stats Expand all files Collapse all files

2
.gitignore vendored
View File

@ -5,5 +5,3 @@
/data /data
backend/api_logs.txt backend/api_logs.txt
*__pycache__* *__pycache__*
.env
poetry.lock

215
backend/api/pipelines.py
View File

@ -1,215 +0,0 @@
def parse_int(args):
try:
return int( args )
except ValueError:
print(f"{args} cannot be cast to int")
raise
def mangrove_by_country_latest():
return """
select * from models_final.final__protected_mangroves_summary_stats_by_country_agg
where year = '2020'
order by cumulative_pixels_diff desc
"""
def bitcoin_business_growth_timeseries(args):
days_ago = parse_int(args["days_ago"])
country_name = args["country_name"]
return f"""
select * from models_final.final__bitcoin_business_growth_by_country
where days_ago <= {days_ago} and country_name = '{country_name}'
order by date
"""
def bitcoin_business_growth_percent_diff_days_ago(args):
days_ago = parse_int(args["days_ago"])
return f"""
with
filtered_data as (
select country_name, date, days_ago, cumulative_value
from models_final.final__bitcoin_business_growth_by_country
where days_ago <= {days_ago}
order by country_name, days_ago desc
),
first_and_last_values as (
select
country_name,
date,
days_ago,
cumulative_value,
first_value(cumulative_value) over (
partition by country_name order by days_ago desc
) as first_value,
first_value(date) over (
partition by country_name order by days_ago desc
) as first_date,
first_value(cumulative_value) over (
partition by country_name order by days_ago
) as last_value,
first_value(date) over (
partition by country_name order by days_ago
) as last_date
from filtered_data
),
diff as (
select
country_name,
date,
first_date,
last_date,
days_ago,
cumulative_value,
first_value,
last_value,
last_value - first_value as difference,
round(
100 * safe_divide((last_value - first_value), first_value), 2
) as percent_difference
from first_and_last_values
)
select *
from diff
where days_ago = 1
order by difference desc
"""
# def bitcoin_business_growth_timeseries(query):
# pipeline = [
# {
# "$match": {
# "days_ago": {"$lte": int(query["days_ago"])},
# "country_name": query["country_name"],
# }
# },
# {
# "$project": {
# "country_name": "$country_name",
# "date": "$date",
# "cumulative_value": "$cumulative_value",
# }
# },
# {"$sort": {"country_name": 1, "days_ago": 1}},
# ]
# return pipeline
# def mangrove_by_country_latest():
# pipeline = [
# {
# "$match": {"year": "2020"},
# },
# ]
# return pipeline
#
#
# def mangrove_by_country_agg(query):
# pipeline = [
# {"$match": {"country_with_parent": query["country_with_parent"]}},
# {
# "$group": {
# "_id": {"country_with_parent": "$country_with_parent", "year": "$year"},
# "total_pixels": {"$sum": "$total_n_pixels"},
# }
# },
# {
# "$project": {
# "_id": 0,
# "country_with_parent": "$_id.country_with_parent",
# "year": "$_id.year",
# "total_pixels": 1,
# }
# },
# {"$sort": {"year": 1}},
# ]
# return pipeline
#
#
# def bitcoin_business_growth_timeseries(query):
# pipeline = [
# {
# "$match": {
# "days_ago": {"$lte": int(query["days_ago"])},
# "country_name": query["country_name"],
# }
# },
# {
# "$project": {
# "country_name": "$country_name",
# "date": "$date",
# "cumulative_value": "$cumulative_value",
# }
# },
# {"$sort": {"country_name": 1, "days_ago": 1}},
# ]
# return pipeline
#
#
# def bitcoin_business_growth_percent_diff_days_ago(query):
pipeline = [
{"$match": {"days_ago": {"$lte": int(query["days_ago"])}}},
{"$sort": {"country_name": 1, "days_ago": 1}},
{
"$group": {
"_id": "$country_name",
"firstvalue": {"$first": "$cumulative_value"},
"lastvalue": {"$last": "$cumulative_value"},
"firstdate": {"$min": "$date"},
"lastdate": {"$max": "$date"},
}
},
{
"$project": {
"country_name": "$_id",
"first_value": "$firstvalue",
"last_value": "$lastvalue",
"difference": {
"$subtract": [
{"$todouble": "$firstvalue"},
{"$todouble": "$lastvalue"},
]
},
"first_date": "$firstdate",
"last_date": "$lastdate",
"percent_difference": {
"$cond": {
"if": {"$eq": [{"$todouble": "$lastvalue"}, 0]},
"then": {
"$cond": {
"if": {"$gt": [{"$todouble": "$firstvalue"}, 0]},
"then": "new",
"else": "none",
}
},
"else": {
"$round": [
{
"$multiply": [
{
"$divide": [
{
"$subtract": [
{"$todouble": "$firstvalue"},
{"$todouble": "$lastvalue"},
]
},
{"$todouble": "$lastvalue"},
]
},
100,
]
}
]
},
}
},
}
},
]
return pipeline
#
#
# def bitcoin_business_growth_latest(query):
# pipeline = [
# {
# "$match": query["filter"],
# },
# {"$sort": {"date": 1}},
# ]
# return pipeline

34
backend/api/postgres_handler.py
View File

@ -1,34 +0,0 @@
from psycopg2.extras import RealDictCursor
import psycopg2, os
class PostgresHandler:
def __init__(self):
self.connection = self.connect_to_pg()
self.cur = self.connection.cursor(cursor_factory=RealDictCursor)
def connect_to_pg(self):
try:
connection = psycopg2.connect(
dbname=os.getenv('PGDATABASE'),
host=os.getenv('PGHOST'),
user=os.getenv('PGUSER'),
password=os.getenv('PGPASSWORD'),
port=os.getenv('PGPORT'),
)
except Exception as e:
message=f"Connection to postgres database failed: {e}"
raise Exception(message)
print(f"Successfully connected to DB")
return connection
def execute_query(self, query):
try:
self.cur.execute(query)
results = self.cur.fetchall()
self.connection.commit()
self.connection.close()
return results
except Exception:
print("Error executing query")
raise

96
backend/api/route.py
View File

@ -1,96 +0,0 @@
from fastapi import APIRouter
from api.postgres_handler import PostgresHandler
import api.pipelines as pipelines
import api.schemas as schemas
from api.schemas import DataSerializer
import json
router = APIRouter()
def parse_args_to_dict(query):
try:
return json.loads(query)
except json.JSONDecodeError as e:
return {"error": f"Invalid JSON: {e}"}
@router.get("/mangrove_by_country_latest")
async def mangrove_by_country_latest():
pipeline = pipelines.mangrove_by_country_latest()
handler = PostgresHandler()
schema = schemas.mangrove_by_country_latest_schema
serializer = DataSerializer(schema)
rawData = handler.execute_query(pipeline)
serializedData = serializer.serialize_many(rawData)
return serializedData
@router.get("/bitcoin_business_growth_timeseries")
async def bitcoin_business_growth_timeseries(query: str):
args = parse_args_to_dict(query)
pipeline = pipelines.bitcoin_business_growth_timeseries(args)
handler = PostgresHandler()
schema = schemas.bitcoin_business_growth_timeseries_schema
serializer = DataSerializer(schema)
rawData = handler.execute_query(pipeline)
serializedData = serializer.serialize_many(rawData)
return serializedData
@router.get("/bitcoin_business_growth_percent_diff")
async def bitcoin_business_growth_percent_diff(query: str):
args = parse_args_to_dict(query)
pipeline = pipelines.bitcoin_business_growth_percent_diff_days_ago(args)
handler = PostgresHandler()
schema = schemas.bitcoin_business_growth_percent_diff_schema
serializer = DataSerializer(schema)
rawData = handler.execute_query(pipeline)
serializedData = serializer.serialize_many(rawData)
return serializedData
# @router.get("/bitcoin_business_growth_percent_diff")
# async def bitcoin_business_growth_percent_diff(query: str):
# query = ast.literal_eval(query)
#
# query = queries.bitcoin_business_growth_percent_diff_days_ago(query)
# handler = PostgresHandler(connection)
#
# schema = schemas.bitcoin_business_growth_percent_diff_schema
# pipeline = pipelines.bitcoin_business_growth_percent_diff_days_ago(query)
# serializer = DataSerializer(schema)
# handler = MongoDBHandler(collection_name)
# rawData = handler.aggregate(pipeline)
# serializedData = serializer.serialize_many(rawData)
# return serializedData
# @router.get("/mangrove_by_country_agg")
# async def mangrove_by_country_agg(query: str):
# query = ast.literal_eval(query)
# db = client.baseddata
# collection_name = db["final__protected_mangroves_summary_stats_by_country_agg"]
# schema = schemas.mangrove_by_country_agg_schema
# pipeline = pipelines.mangrove_by_country_agg(query)
# serializer = DataSerializer(schema)
# handler = MongoDBHandler(collection_name)
# rawData = handler.aggregate(pipeline)
# serializedData = serializer.serialize_many(rawData)
# return serializedData
#
# @router.get("/bitcoin_business_growth_timeseries")
# async def bitcoin_business_growth_timeseries(query: str):
# query = ast.literal_eval(query)
# db = client.baseddata
# collection_name = db["final__bitcoin_business_growth_by_country"]
# schema = schemas.bitcoin_business_growth_timeseries_schema
# pipeline = pipelines.bitcoin_business_growth_timeseries(query)
# serializer = DataSerializer(schema)
# handler = MongoDBHandler(collection_name)
# rawData = handler.aggregate(pipeline)
# serializedData = serializer.serialize_many(rawData)
# return serializedData

42
backend/api/schemas.py
View File

@ -1,42 +0,0 @@
def mangrove_by_country_latest_schema(data):
return {
"country_with_parent": str(data["country_with_parent"]),
"original_pixels": int(data["original_pixels"]),
"total_n_pixels": int(data["total_n_pixels"]),
"cumulative_pixels_diff": int(data["cumulative_pixels_diff"]),
"cumulative_pct_diff": float(data["cumulative_pct_diff"]),
}
def mangrove_by_country_agg_schema(data):
return {
"country_with_parent": str(data["country_with_parent"]),
"year": int(data["year"]),
"total_pixels": int(data["total_pixels"])
}
def bitcoin_business_growth_percent_diff_schema(data):
return {
"country_name": str(data["country_name"]),
"date_range": str(f'{data["first_date"]} to {data["last_date"]}'),
"first_value": int(data["first_value"]),
"last_value": int(data["last_value"]),
"difference": int(data["difference"]),
"percent_difference": str(data["percent_difference"])
}
def bitcoin_business_growth_timeseries_schema(data):
return {
"country_name": str(data["country_name"]),
"date": data["date"],
"cumulative_value": int(data["cumulative_value"])
}
class DataSerializer:
def __init__(self, schema_func):
self.schema_func = schema_func
def serialize_one(self, data) -> dict:
return self.schema_func(dict( data ))
def serialize_many(self, data_list) -> list:
return [self.serialize_one(data) for data in data_list]

210
backend/app.py Normal file
View File

@ -0,0 +1,210 @@
from flask import Flask, g, jsonify, request, json, Response, send_from_directory, abort
from flask_cors import CORS
import orjson, os
import pandas as pd
import datetime
import time
app = Flask(__name__)
CORS(app)
FILES_DIRECTORY = "../data/"
@app.before_request
def start_timer():
g.start = time.time()
@app.after_request
def log(response):
now = time.time()
duration = round(now - g.start, 4)
dt = datetime.datetime.fromtimestamp(now).strftime("%Y-%m-%d %H:%M:%S")
log_entry = {
"timestamp": dt,
"duration": duration,
"method": request.method,
"url": request.url,
"status": response.status_code,
"remote_addr": request.access_route[-1],
"user_agent": request.user_agent.string,
}
log_line = ",".join(f"{key}={value}" for key, value in log_entry.items())
with open("api_logs.txt", "a") as f:
f.write(log_line + "\n")
return response
@app.route("/bitcoin_business_growth_by_country", methods=["GET"])
def business_growth():
today = datetime.datetime.today()
# Parse args from request
latest_date = request.args.get("latest_date")
country_names = request.args.get("countries")
cumulative_period_type = request.args.get("cumulative_period_type")
# Open json locally
with open("../data/final__bitcoin_business_growth_by_country.json", "rb") as f:
data = orjson.loads(f.read())
# Filter based on args
if latest_date:
latest_date_bool = latest_date == "true"
filtered_data = [
item for item in data if item["latest_date"] == latest_date_bool
]
else:
filtered_data = data
if country_names:
countries = [name.strip() for name in country_names.split(",")]
filtered_data = [
item for item in filtered_data if item["country_name"] in countries
]
if cumulative_period_type == "1 day":
delta = today - datetime.timedelta(days=2)
filtered_data = [
item
for item in filtered_data
if item["cumulative_period_type"] == cumulative_period_type
and delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d")
]
elif cumulative_period_type == "7 day":
delta = today - datetime.timedelta(days=8)
filtered_data = [
item
for item in filtered_data
if item["cumulative_period_type"] == cumulative_period_type
and delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d")
]
elif cumulative_period_type == "28 day":
delta = today - datetime.timedelta(days=29)
filtered_data = [
item
for item in filtered_data
if item["cumulative_period_type"] == cumulative_period_type
and delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d")
]
elif cumulative_period_type == "365 day":
delta = today - datetime.timedelta(days=366)
filtered_data = [
item
for item in filtered_data
if item["cumulative_period_type"] == cumulative_period_type
and delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d")
]
# Sort by date
sorted_data = sorted(filtered_data, key=lambda x: x["date"], reverse=False)
# Return json
return Response(json.dumps(sorted_data), mimetype="application/json")
@app.route("/get_json/<filename>", methods=["GET"])
def get_json(filename):
period = request.args.get("period")
today = datetime.datetime.today()
file_path = os.path.join(FILES_DIRECTORY, filename)
if not os.path.isfile(file_path):
abort(404)
with open(file_path, "r") as f:
data = orjson.loads(f.read())
if period == "last 7 days":
delta = today - datetime.timedelta(days=7)
filtered_data = [
item
for item in data
if delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d") <= today
]
sorted_data = sorted(filtered_data, key=lambda x: x["date"])
elif period == "last 28 days":
delta = today - datetime.timedelta(days=28)
filtered_data = [
item
for item in data
if delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d") <= today
]
sorted_data = sorted(filtered_data, key=lambda x: x["date"])
elif period == "last 365 days":
delta = today - datetime.timedelta(days=365)
filtered_data = [
item
for item in data
if delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d") <= today
]
sorted_data = sorted(filtered_data, key=lambda x: x["date"])
elif period == "last 2 years":
delta = today - datetime.timedelta(days=730)
filtered_data = [
item
for item in data
if delta <= datetime.datetime.strptime(item["date"], "%Y-%m-%d") <= today
]
sorted_data = sorted(filtered_data, key=lambda x: x["date"])
else:
sorted_data = sorted(data, key=lambda x: x["date"])
return jsonify(sorted_data)
@app.route("/mangrove_data/<method>", methods=["GET"])
def mangrove_data(method):
with open("../data/dev/final__wdpa_pid_mangrove_diff_stats.json", "rb") as f:
data = orjson.loads(f.read())
if method == "countries":
df = pd.read_json(json.dumps(data))
countries = df[["year", "country", "n_pixels", "diff", "cumulative_diff"]]
countriesAgg = countries.groupby(["year", "country"]).agg(
{"n_pixels": "sum", "diff": "sum", "cumulative_diff": "sum"}
)
countriesAgg["year0_pixels"] = (
countriesAgg["n_pixels"] - countriesAgg["cumulative_diff"]
)
countriesAgg["pct_diff"] = (
100
* (countriesAgg["n_pixels"] - countriesAgg["year0_pixels"])
/ countriesAgg["year0_pixels"]
).round(2)
countriesLatest = countriesAgg.loc[[2020]].reset_index().set_index("country")
return Response(
countriesLatest.to_json(orient="index"), mimetype="application/json"
)
@app.route("/download/<filename>", methods=["GET"])
def download_file(filename):
try:
return send_from_directory(FILES_DIRECTORY, filename, as_attachment=True)
except FileNotFoundError:
abort(404)
@app.route("/cog", methods=["GET"])
def serve_cog():
year = request.args.get("year")
pid = request.args.get("pid") # change this line
dir = f"{FILES_DIRECTORY}/cog/{year}/"
try:
return send_from_directory(dir, f"{pid}.tif", as_attachment=True)
except FileNotFoundError:
abort(404)
if __name__ == "__main__":
app.run()

6
backend/main.py
View File

@ -1,6 +0,0 @@
from fastapi import FastAPI
from api.route import router
app = FastAPI()
app.include_router(router)

6
content/data-lab/global-business-growth.md
View File

@ -15,10 +15,8 @@ You can select the growth period of interest from the drop-down, which updates t
The chart always reflects the countries selected in the table. The chart always reflects the countries selected in the table.
<br/> <br/>
{{< dropdown_filter id="days_ago_dropdown_filter" id_filter="days_ago" options="1 day:1,7 day:7,28 day:28,1 year:365,5 year:1826,10 year:3652,all time:10000" default_selection="7 day" targets="bitcoin-business-growth-chart bitcoin-business-growth-table" >}} {{< chart src="/js/bitcoin-business-growth-chart.js" >}}
{{< table id="bitcoin-business-growth-table" endpoint="bitcoin_business_growth_percent_diff" headers="{'country_name': 'Country', 'date_range': 'Date Range', 'first_value': 'Previous #', 'last_value': 'Current #', 'difference': 'Diff', 'percent_difference': '% Diff'}" maxHeight="400px" sortable="true" valueId="country_name" selectableRows="multi" targets="bitcoin-business-growth-chart" defaultFirstSelected="true" >}} {{< table src="/js/bitcoin-business-growth-table.js" >}}
{{< chart id="bitcoin-business-growth-chart" endpoint="bitcoin_business_growth_timeseries" chartType="line" xAxisField="date" yAxisField="cumulative_value" scaleChart=true >}}
#### Attribution and License #### Attribution and License
Data obtained from © [OpenStreetMap](https://www.openstreetmap.org/copyright) Data obtained from © [OpenStreetMap](https://www.openstreetmap.org/copyright)

5
content/data-lab/protected-mangroves.md
View File

@ -9,8 +9,5 @@ tags: ["Bitcoin", "Stats"]
script: "/js/mangrove-map.js" script: "/js/mangrove-map.js"
--- ---
{{< table id="mangrove_countries" endpoint="mangrove_by_country_latest" headers="{'country_with_parent': 'Country', 'original_pixels': '1996 Cover', 'total_n_pixels': '2020 Cover', 'cumulative_pixels_diff': 'Diff', 'cumulative_pct_diff': '% Diff'}" maxHeight="400px" sortable="true" valueId="country_with_parent" selectableRows="single" defaultFirstSelected="true" >}} {{< table id="mangrove_countries" endpoint="mangrove_data/countries" tableKey="country" headers="{'country': 'Country', 'cumulative_diff': 'Diff', 'pct_diff': '% Diff'}" maxHeight="400px" sortable="true">}}
{{< chart id="mangrove_countries" endpoint="mangrove_by_country_agg" chartType="bar" xAxisField="year" yAxisField="total_pixels" scaleChart=true >}}
{{< map id="map" style="https://tiles.semitamaps.com/styles/maptiler-basic/style.json">}} {{< map id="map" style="https://tiles.semitamaps.com/styles/maptiler-basic/style.json">}}
{{< chart id="mangrove-country-timeseries-chart" endpoint="mangrove_country_timeseries" chartType="line" xAxisField="date" yAxisField="n_pixels" scaleChart=true >}}

2
hugo.toml
View File

@ -3,7 +3,7 @@ languageCode = 'en-gb'
title = 'Based Data' title = 'Based Data'
[params] [params]
apiURL = 'http://localhost:8000' apiURL = 'http://localhost:5000'
[markup.highlight] [markup.highlight]
pygmentsUseClasses = false pygmentsUseClasses = false

100
layouts/partials/chart.html
View File

@ -1,96 +1,8 @@
<script src="https://cdn.jsdelivr.net/npm/echarts@5.5.1/dist/echarts.min.js"></script> <script src="https://cdn.jsdelivr.net/npm/echarts@5.5.1/dist/echarts.min.js"></script>
<script> <section class="chart-container">
chartData = []; <div id="chart-modifiers"></div>
function createChart( <div id="chart">
id, <script src="{{ .src }}"></script>
endpoint, </div>
chartType, </section>
xAxisField,
yAxisField,
sortField = null,
scaleChart = false,
) {
async function fetchDataForChart(query, valueId) {
try {
const apiEndpoint = `${apiURL}/${endpoint}?${query}`;
const response = await fetch(apiEndpoint);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const fetchedData = await response.json();
const newData = fetchedData.reduce((acc, item) => {
const objectId = item[valueId];
if (!acc[objectId]) {
acc[objectId] = [];
}
acc[objectId].push([item[xAxisField], item[yAxisField]]);
return acc;
}, {});
chartData = { ...chartData, ...newData };
updateChart();
} catch (error) {
console.error("Fetching data failed:", error);
}
}
function updateChart() {
console.log(chartData);
let chartDataMap = new Map();
for (let objectId in chartData) {
chartDataMap.set(objectId, chartData[objectId]);
}
var chartDom = document.getElementById(`${id}`);
var myChart = echarts.init(chartDom);
var option = {
tooltip: {
...tooltip,
valueFormatter(value, index) {
return nFormatter(value, 0);
},
},
xAxis: {
type: "time",
},
yAxis: {
scale: scaleChart,
type: "value",
},
series: Array.from(chartDataMap.entries()).map(([name, data]) => ({
name,
type: chartType,
data,
showSymbol: false,
})),
};
myChart.setOption(option, true);
}
// listen for filter events for this target
document.addEventListener("filterChange", function (event) {
tableId = document.getElementById(id).id;
console.log(event.detail);
eventDetail = event.detail;
if (eventDetail.filterActions.includes("refresh")) {
chartData = [];
updateChart();
} else {
if (eventDetail.filterTargets.includes(tableId)) {
if (eventDetail.filterActions.includes("selected")) {
valueId = eventDetail.filterId;
let selectedRow = {
[valueId]: eventDetail.filterValue,
};
query = queryConstructor(selectedRow);
fetchDataForChart(query, valueId);
} else {
delete chartData[eventDetail.filterValue];
updateChart();
}
}
}
});
}
</script>
<script src="/js/chart-params.js"></script> <script src="/js/chart-params.js"></script>

2
layouts/partials/head.html
View File

@ -8,8 +8,6 @@
<link rel="stylesheet" href="/css/toc.css" type="text/css" media="all" /> <link rel="stylesheet" href="/css/toc.css" type="text/css" media="all" />
<link rel="stylesheet" href="/css/articles.css" type="text/css" media="all" /> <link rel="stylesheet" href="/css/articles.css" type="text/css" media="all" />
<link rel="stylesheet" href="/css/charts.css" type="text/css" media="all" /> <link rel="stylesheet" href="/css/charts.css" type="text/css" media="all" />
<script src="/js/lib/sorttable.js"></script>
<script src="/js/lib/helper-functions.js"></script>
<link <link
rel="stylesheet" rel="stylesheet"
href="/css/codeblock.css" href="/css/codeblock.css"

142
layouts/partials/table.html
View File

@ -1,142 +0,0 @@
<script>
function createTable(
endpoint,
id,
headers,
maxHeight,
sortable,
valueId,
selectableRows,
filterTargets,
defaultFirstSelected,
) {
async function fetchDataForTable(query) {
try {
const apiEndpoint = `${apiURL}/${endpoint}?${query}`;
const response = await fetch(apiEndpoint);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const fetchedData = await response.json();
data = fetchedData;
generateTable(data);
} catch (error) {
console.error("Fetching data failed:", error);
}
}
function generateTable(data) {
const jsonTableContainer = document.getElementById(`${id}--container`);
jsonTableContainer.className = "jsonTableContainer";
jsonTableContainer.innerHTML = "";
jsonTableContainer.style.maxHeight = maxHeight;
tableHeaderNames = Object.values(headers);
tableHeaderKeys = Object.keys(headers);
const table = document.createElement("table");
table.id = `${id}`;
const thead = document.createElement("thead");
const tbody = document.createElement("tbody");
const headerRow = document.createElement("tr");
tableHeaderNames.forEach((header) => {
const th = document.createElement("th");
th.textContent = header;
headerRow.appendChild(th);
});
thead.appendChild(headerRow);
table.appendChild(thead);
for (const key in data) {
const row = document.createElement("tr");
row.value = data[key][valueId];
tableHeaderKeys.forEach((columnName) => {
const td = document.createElement("td");
const div = document.createElement("div");
div.id = "scrollable";
div.textContent = data[key][columnName];
td.appendChild(div);
row.appendChild(td);
tbody.appendChild(row);
});
}
table.appendChild(thead);
table.appendChild(tbody);
jsonTableContainer.appendChild(table);
// sortable
if (sortable == "true") {
table.className = "sortable";
sorttable.makeSortable(document.getElementById(`${id}`));
}
if (selectableRows === "multi" || selectableRows === "single") {
const rows = table.getElementsByTagName("tr");
for (let i = 1; i < rows.length; i++) {
rows[i].addEventListener("click", function () {
if (selectableRows === "multi") {
this.classList.toggle("selected");
if (this.classList.contains("selected")) {
const event = new CustomEvent("filterChange", {
detail: {
filterId: valueId,
filterValue: this.value,
filterActions: ["selected"],
filterTargets: filterTargets,
},
});
document.dispatchEvent(event);
} else {
const event = new CustomEvent("filterChange", {
detail: {
filterId: valueId,
filterValue: this.value,
filterActions: ["deselected"],
filterTargets: filterTargets,
},
});
document.dispatchEvent(event);
}
} else if (selectableRows === "single") {
if (this.classList.contains("selected")) {
this.classList.remove("selected");
} else {
for (let j = 1; j < rows.length; j++) {
rows[j].classList.remove("selected");
}
this.classList.add("selected");
}
}
});
if (defaultFirstSelected == true) {
if (i == 1) {
rows[i].classList.add("selected");
const event = new CustomEvent("filterChange", {
detail: {
filterId: valueId,
filterValue: rows[i].value,
filterActions: ["selected"],
filterTargets: filterTargets,
},
});
document.dispatchEvent(event);
}
}
}
}
}
// listen for filter events for this target
document.addEventListener("filterChange", function (event) {
tableId = document.getElementById(id).id;
if (event.detail.filterTargets.includes(tableId)) {
query = queryConstructor();
fetchDataForTable(query);
}
});
query = queryConstructor();
fetchDataForTable(query);
}
</script>

12
layouts/shortcodes/chart.html
View File

@ -1,10 +1,2 @@
{{ partial "chart.html" }} {{ $id := .Get "src" | md5 }} {{ partial "chart.html" (dict "src" (.Get "src")
<section class = 'chart-container'> "id" $id) }}
<div class = "chart" id='{{ .Get "id" }}'>
<script>
document.addEventListener("DOMContentLoaded", function () {
createChart(id={{ .Get "id" }}, endpoint={{ .Get "endpoint" }}, chartType={{ .Get "chartType" }}, xAxisField={{ .Get "xAxisField" }}, yAxisField={{ .Get "yAxisField" }}, sortField={{ .Get "sortField" }}, scaleChart={{ .Get "scaleChart" }})
});
</script>
</div>
</section>

29
layouts/shortcodes/dropdown_filter.html
View File

@ -1,29 +0,0 @@
{{ $id := .Get "id" }}
{{ $default_selection := .Get "default_selection" }}
{{ $options := .Get "options" }}
{{ $options_split := split $options "," }}
<div class="dropdown-filter-container">
<select class="filter dropdown-filter" id="{{ $id }}" idFilter='{{ .Get "id_filter" }}' onchange="dispatchDropdownEvent(this)">
{{ range $options_split }}
{{ $parts := split . ":" }}
{{ $key := index $parts 0 }}
{{ $value := index $parts 1 }}
<option value="{{ $value }}" {{ if eq $key $default_selection }}selected{{ end }}>{{ $key }}</option>
{{ end }}
</select>
<script>
function dispatchDropdownEvent(selectElement) {
const event = new CustomEvent('filterChange', {
detail: {
filterId: '{{ .Get "id_filter" }}',
filterValue: selectElement.value,
filterActions: ["refresh"],
filterTargets: '{{ .Get "targets" }}'.split(" ")
}
});
document.dispatchEvent(event);
}
</script>
</div>

71
layouts/shortcodes/table.html
View File

@ -1,8 +1,67 @@
{{ partial "table.html" }}
<div id = '{{ .Get "id" }}--container'>
<script> <script>
document.addEventListener("DOMContentLoaded", function () { async function fetchDataForTable() {
createTable({{ .Get "endpoint" }}, {{ .Get "id" }}, {{ .Get "headers" | safeJS }}, {{ .Get "maxHeight" }}, {{ .Get "sortable" }}, {{ .Get "valueId" }}, {{ .Get "selectableRows" }}, '{{ .Get "targets" }}'.split(" "), {{ .Get "defaultFirstSelected" | safeJS }}) try {
}); const apiEndpoint = `${apiURL}/{{ .Get "endpoint" }}`;
const response = await fetch(apiEndpoint);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const fetchedData = await response.json();
data = fetchedData;
console.log(data);
createTable(data);
} catch (error) {
console.error("Fetching data failed:", error);
}
}
function createTable(data) {
const jsonTableContainer = document.getElementById("jsonTableContainer");
jsonTableContainer.innerHTML = "";
jsonTableContainer.style.maxHeight = "{{ .Get `maxHeight` }}"
tableHeaderNames = Object.values({{ .Get `headers` | safeJS }});
tableHeaderKeys = Object.keys({{ .Get `headers` | safeJS }});
const table = document.createElement("table");
table.id = "{{ .Get `id` }}"
const thead = document.createElement("thead");
const tbody = document.createElement("tbody");
const headerRow = document.createElement("tr");
tableHeaderNames.forEach((header) => {
const th = document.createElement("th");
th.textContent = header;
headerRow.appendChild(th);
});
thead.appendChild(headerRow);
table.appendChild(thead);
for (const key in data) {
const row = document.createElement('tr');
tableHeaderKeys.forEach((columnName) => {
const td = document.createElement("td");
const div = document.createElement("div");
div.id = "scrollable";
if (columnName == "{{ .Get `tableKey` }}") {
div.textContent = key;
} else {
div.textContent = data[key][columnName];
};
td.appendChild(div);
row.appendChild(td);
tbody.appendChild(row);
});
}
table.appendChild(thead);
table.appendChild(tbody);
jsonTableContainer.appendChild(table)
{{ if eq (.Get "sortable") "true" }}
table.className = "sortable"
sorttable.makeSortable(document.getElementById("{{ .Get `id` }}"));
{{ end }}
table.className = ""
}
</script> </script>
</div> <div id="jsonTableContainer"></div>
<script src="/js/lib/sorttable.js"></script>

22
pyproject.toml
View File

@ -1,22 +0,0 @@
[tool.poetry]
name = "baseddata-io"
version = "0.1.0"
description = ""
authors = ["Sam <samual.shop@proton.me>"]
readme = "README.md"
packages = [{include = "baseddata"}]
package-mode = false
[virtualenvs]
in-project = true
[tool.poetry.dependencies]
python = "^3.11"
fastapi = "^0.115.4"
uvicorn = "^0.32.0"
psycopg2 = "^2.9.10"
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

24
shell.nix
View File

@ -1,19 +1,20 @@
{pkgs ? import <nixpkgs> {}}: { pkgs ? import <nixpkgs> { } }:
pkgs.mkShell pkgs.mkShell
{ {
nativeBuildInputs = with pkgs; [ nativeBuildInputs = with pkgs; [
python312Packages.flask
python312Packages.flask-cors
python312Packages.requests
python312Packages.pandas
python312Packages.orjson
hugo hugo
python311 tmux
poetry
]; ];
NIX_LD_LIBRARY_PATH=/run/current-system/sw/share/nix-ld/lib; shellHook = ''
NIX_LD=/run/current-system/sw/share/nix-ld/lib/ld.so;
shellHook = ''
${pkgs.cowsay}/bin/cowsay "Welcome to the baseddata.io development environment!" | ${pkgs.lolcat}/bin/lolcat ${pkgs.cowsay}/bin/cowsay "Welcome to the baseddata.io development environment!" | ${pkgs.lolcat}/bin/lolcat
export LD_LIBRARY_PATH=$NIX_LD_LIBRARY_PATH
source .env
source .venv/bin/activate
get_session=$(tmux list-session | grep "baseddata") get_session=$(tmux list-session | grep "baseddata")
if [ -z "$get_session" ]; if [ -z "$get_session" ];
@ -21,8 +22,9 @@ pkgs.mkShell
tmux new-session -d -s baseddata tmux new-session -d -s baseddata
tmux split-window -h tmux split-window -h
tmux send-keys -t 0 "hugo server" C-m tmux send-keys -t 0 "hugo server" C-m
tmux send-keys -t 1 "cd backend && uvicorn main:app --reload" C-m tmux send-keys -t 1 "cd backend && python app.py" C-m
echo "Baseddata running in dev tmux shell" echo "Baseddata running in dev tmux shell"
fi fi
''; '';
} }

26
static/css/charts.css
View File

@ -1,12 +1,24 @@
/* Charts */ /* Charts */
.chart-container { .chart-flex-container {
display: flex; display: flex;
/* height: 600px; */
aspect-ratio: 1 / 1;
} }
.chart { .chart-flex-container article {
display: flex; flex: 1;
height: 100%; }
width: 100%;
#chart {
width: 100%;
aspect-ratio: 16 / 9;
}
.chart-container {
margin-top: 20px;
display: flex;
justify-content: center;
flex-direction: column;
}
#chart-modifiers {
display: flex;
} }

13
static/css/tables.css
View File

@ -1,13 +1,9 @@
/* Tables */ /* Tables */
.jsonTableContainer { #jsonTableContainer {
display: flex; display: flex;
overflow-y: auto; overflow-y: auto;
} }
table.sortable th:not(.sorttable_sorted):not(.sorttable_sorted_reverse):not(.sorttable_nosort):after {
content: " \25B4\25BE"
}
table { table {
width: 100%; width: 100%;
border-collapse: collapse; border-collapse: collapse;
@ -53,10 +49,3 @@ tr:nth-child(odd) {
background-color: var(--table-odd-row-bg-color); background-color: var(--table-odd-row-bg-color);
font-size: var(--table-row-font-size); font-size: var(--table-row-font-size);
} }
tr:hover {
background-color: #f5f5f5;
}
tr.selected {
background-color: #d1ecf1;
}

16
static/js/lib/helper-functions.js
View File

@ -1,16 +0,0 @@
function queryConstructor(customFilters = {}) {
let filters = document.querySelectorAll(".filter");
let queryObject = {};
Object.assign(queryObject, customFilters);
filters.forEach((filter) => {
const filterId = filter.getAttribute("idFilter");
const filterValue = filter.value;
queryObject[filterId] = filterValue;
});
let queryString = `query=${JSON.stringify(queryObject)}`;
return queryString;
}

3
static/js/mangrove-map.js
View File

@ -15,3 +15,6 @@ map.on("load", () => {
}); });
}); });
document.addEventListener("DOMContentLoaded", function () {
fetchDataForTable();
});