sam
/
baseddata.io
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: sam:5930fe90a10c0213091a04e5e4c1cb9ee41f83c3
Branches Tags
sam:master
sam:dev
sam:new-backend
sam:data-downloads
..
compare: sam:e9cd27d5c441402da57c092f8e1827a31dc8c8e2
Branches Tags
sam:dev
sam:new-backend
sam:master
sam:data-downloads

No commits in common. "5930fe90a10c0213091a04e5e4c1cb9ee41f83c3" and "e9cd27d5c441402da57c092f8e1827a31dc8c8e2" have entirely different histories.
5930fe90a1 ... e9cd27d5c4

15 changed files with 137 additions and 300 deletions
Show Stats Expand all files Collapse all files

94
backend/models/pipelines.py
View File

@ -27,97 +27,3 @@ def mangrove_by_country_agg(query):
{"$sort": {"year": 1}},
]
return pipeline
def bitcoin_business_growth_timeseries(query):
pipeline = [
{
"$match": {
"days_ago": {"$lte": int( query["days_ago"] )},
"country_name": query["country_name"],
}
},
{
"$project": {
"country_name": "$country_name",
"date": "$date",
"cumulative_value": "$cumulative_value",
}
},
{"$sort": {"country_name": 1, "days_ago": 1}},
]
return pipeline
def bitcoin_business_growth_percent_diff_days_ago(query):
pipeline = [
{"$match": {"days_ago": {"$lte": int( query["days_ago"] )}}},
{"$sort": {"country_name": 1, "days_ago": 1}},
{
"$group": {
"_id": "$country_name",
"firstValue": {"$first": "$cumulative_value"},
"lastValue": {"$last": "$cumulative_value"},
"firstDate": {"$min": "$date"},
"lastDate": {"$max": "$date"},
}
},
{
"$project": {
"country_name": "$_id",
"first_value": "$firstValue",
"last_value": "$lastValue",
"difference": {
"$subtract": [
{"$toDouble": "$firstValue"},
{"$toDouble": "$lastValue"},
]
},
"first_date": "$firstDate",
"last_date": "$lastDate",
"percent_difference": {
"$cond": {
"if": {"$eq": [{"$toDouble": "$lastValue"}, 0]},
"then": {
"$cond": {
"if": {"$gt": [{"$toDouble": "$firstValue"}, 0]},
"then": "new",
"else": "none",
}
},
"else": {
"$round": [
{
"$multiply": [
{
"$divide": [
{
"$subtract": [
{"$toDouble": "$firstValue"},
{"$toDouble": "$lastValue"},
]
},
{"$toDouble": "$lastValue"},
]
},
100,
]
}
]
},
}
},
}
},
]
return pipeline
def bitcoin_business_growth_latest(query):
pipeline = [
{
"$match": query["filter"],
},
{"$sort": {"date": 1}},
]
return pipeline

21
backend/routes/route.py
View File

@ -34,26 +34,13 @@ async def mangrove_by_country_agg(query: str):
serializedData = serializer.serialize_many(rawData)
return serializedData
@router.get("/bitcoin_business_growth_percent_diff")
async def bitcoin_business_growth_percent_diff(query: str):
@router.get("/bitcoin_business_growth")
async def bitcoin_business_growth(query: str):
query = ast.literal_eval(query)
db = client.baseddata
collection_name = db["final__bitcoin_business_growth_by_country"]
schema = schemas.bitcoin_business_growth_percent_diff_schema
pipeline = pipelines.bitcoin_business_growth_percent_diff_days_ago(query)
serializer = DataSerializer(schema)
handler = MongoDBHandler(collection_name)
rawData = handler.aggregate(pipeline)
serializedData = serializer.serialize_many(rawData)
return serializedData
@router.get("/bitcoin_business_growth_timeseries")
async def bitcoin_business_growth_timeseries(query: str):
query = ast.literal_eval(query)
db = client.baseddata
collection_name = db["final__bitcoin_business_growth_by_country"]
schema = schemas.bitcoin_business_growth_timeseries_schema
pipeline = pipelines.bitcoin_business_growth_timeseries(query)
schema = schemas.bitcoin_business_growth
pipeline = pipelines.bitcoin_business_growth(query)
serializer = DataSerializer(schema)
handler = MongoDBHandler(collection_name)
rawData = handler.aggregate(pipeline)

20
backend/schema/schemas.py
View File

@ -1,6 +1,3 @@
def dt_to_date(datetime):
return datetime.date()
def mangrove_by_country_latest_schema(data):
return {
"country_with_parent": str(data["country_with_parent"]),
@ -17,21 +14,12 @@ def mangrove_by_country_agg_schema(data):
"total_pixels": int(data["total_pixels"])
}
def bitcoin_business_growth_percent_diff_schema(data):
def bitcoin_business_growth_schema(data):
return {
"country_name": str(data["country_name"]),
"first_value": int(data["first_value"]),
"date_range": str(f'{dt_to_date(data["first_date"])} to {dt_to_date(data["last_date"])}'),
"last_value": int(data["last_value"]),
"difference": int(data["difference"]),
"percent_difference": str(data["percent_difference"])
}
def bitcoin_business_growth_timeseries_schema(data):
return {
"country_name": str(data["country_name"]),
"date": dt_to_date(data["date"]),
"cumulative_value": int(data["cumulative_value"])
"cumulative_current_value": str(data["cumulative_current_value"]),
"year": int(data["year"]),
"total_pixels": int(data["total_pixels"])
}
class DataSerializer:

6
content/data-lab/global-business-growth.md
View File

@ -15,10 +15,8 @@ You can select the growth period of interest from the drop-down, which updates t
The chart always reflects the countries selected in the table.
<br/>
{{< dropdown_filter id="days_ago" options="1 day:1,7 day:7,28 day:28,1 year:365,5 year:1826,10 year:3652,all time:10000" default_selection="7 day" >}}
{{< table id="bitcoin_business_growth" endpoint="bitcoin_business_growth_percent_diff" headers="{'country_name': 'Country', 'date_range': 'Date Range', 'last_value': 'Previous #', 'first_value': 'Current #', 'difference': 'Diff', 'percent_difference': '% Diff'}" maxHeight="400px" sortable="true" valueId="country_name" selectableRows="multi" >}}
{{< chart id="bitcoin_business_growth" endpoint="bitcoin_business_growth_timeseries" chartType="line" xAxisField="date" yAxisField="cumulative_value" scaleChart=true >}}
{{< chart src="/js/bitcoin-business-growth-chart.js" >}}
{{< table src="/js/bitcoin-business-growth-table.js" >}}
#### Attribution and License
Data obtained from © [OpenStreetMap](https://www.openstreetmap.org/copyright)

2
content/data-lab/protected-mangroves.md
View File

@ -10,5 +10,5 @@ script: "/js/mangrove-map.js"
---
{{< table id="mangrove_countries" endpoint="mangrove_by_country_latest" headers="{'country_with_parent': 'Country', 'original_pixels': '1996 Cover', 'total_n_pixels': '2020 Cover', 'cumulative_pixels_diff': 'Diff', 'cumulative_pct_diff': '% Diff'}" maxHeight="400px" sortable="true" valueId="country_with_parent" selectableRows="single" >}}
{{< chart id="mangrove_countries" endpoint="mangrove_by_country_agg" chartType="bar" xAxisField="year" yAxisField="total_pixels" scaleChart=true >}}
{{< chart id="mangrove_countries" endpoint="mangrove_by_country_agg" chartType="bar" xAxisField="year" yAxisField="total_pixels" sortField="year" scaleChart=true >}}
{{< map id="map" style="https://tiles.semitamaps.com/styles/maptiler-basic/style.json">}}

33
layouts/partials/chart.html
View File

@ -1,4 +1,7 @@
<script src="https://cdn.jsdelivr.net/npm/echarts@5.5.1/dist/echarts.min.js"></script>
<section class="chart-container">
<div id="chart-modifiers"></div>
<div id="chart">
<script>
chartData = [];
function createChart(
@ -12,7 +15,7 @@
) {
async function fetchDataForChart(query, valueId) {
try {
const apiEndpoint = `${apiURL}/${endpoint}?${query}`;
const apiEndpoint = `${apiURL}/${endpoint}?query=${query}`;
const response = await fetch(apiEndpoint);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
@ -39,18 +42,17 @@
for (let objectId in chartData) {
chartDataMap.set(objectId, chartData[objectId]);
}
var chartDom = document.getElementById(`${id}--chart`);
var chartDom = document.getElementById("chart");
var myChart = echarts.init(chartDom);
// // Sort the data by year
// if (sortField) {
// data.sort((a, b) => a[sortField] - b[sortField]);
// }
var option = {
tooltip: {
...tooltip,
valueFormatter(value, index) {
return nFormatter(value, 0);
},
},
xAxis: {
type: "time",
type: "category",
},
yAxis: {
scale: scaleChart,
@ -66,7 +68,6 @@
myChart.setOption(option, true);
}
document.addEventListener("rowSelected", (event) => {
console.log(
"Row selected:",
@ -75,12 +76,8 @@
event.detail.valueId,
event.detail.selectableRows,
);
valueId = event.detail.valueId
let selectedRow = {
[valueId]: event.detail.row.value,
};
query = queryConstructor(selectedRow);
fetchDataForChart(query, valueId);
query = `{'country_with_parent': '${event.detail.row.value}'}`;
fetchDataForChart(query, event.detail.valueId);
});
document.addEventListener("rowDeselected", (event) => {
@ -90,8 +87,10 @@
event.detail.row.value,
);
delete chartData[event.detail.row.value];
updateChart();
updateChart()
});
}
</script>
</div>
</section>
<script src="/js/chart-params.js"></script>

1
layouts/partials/head.html
View File

@ -9,7 +9,6 @@
<link rel="stylesheet" href="/css/articles.css" type="text/css" media="all" />
<link rel="stylesheet" href="/css/charts.css" type="text/css" media="all" />
<script src="/js/lib/sorttable.js"></script>
<script src="/js/lib/helper-functions.js"></script>
<link
rel="stylesheet"
href="/css/codeblock.css"

29
layouts/partials/table.html
View File

@ -8,9 +8,9 @@
valueId,
selectableRows,
) {
async function fetchDataForTable(query) {
async function fetchDataForTable() {
try {
const apiEndpoint = `${apiURL}/${endpoint}?${query}`;
const apiEndpoint = `${apiURL}/${endpoint}`;
const response = await fetch(apiEndpoint);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
@ -25,7 +25,7 @@
function generateTable(data) {
const jsonTableContainer = document.getElementById(
`${id}--table-container`,
`${id}-table-container`,
);
jsonTableContainer.className = "jsonTableContainer";
jsonTableContainer.innerHTML = "";
@ -35,7 +35,7 @@
tableHeaderKeys = Object.keys(headers);
const table = document.createElement("table");
table.id = `${id}--table`;
table.id = id;
const thead = document.createElement("thead");
const tbody = document.createElement("tbody");
const headerRow = document.createElement("tr");
@ -89,6 +89,7 @@
currentlySelectedRow = this.classList.contains("selected")
? this
: null;
} else {
// Multi-select behavior
this.classList.toggle("selected");
@ -99,11 +100,7 @@
? "rowSelected"
: "rowDeselected",
{
detail: {
row: this,
selectableRows: selectableRows,
valueId: valueId,
},
detail: { row: this, selectableRows: selectableRows, valueId: valueId},
},
);
@ -111,21 +108,13 @@
});
}
}
jsonTableContainer.appendChild(table);
// sortable
jsonTableContainer.appendChild(table);
if (sortable == "true") {
table.className = "sortable";
sorttable.makeSortable(document.getElementById(`${id}--table`));
sorttable.makeSortable(document.getElementById(id));
}
}
document.addEventListener("filterChange", function (event) {
query = queryConstructor()
fetchDataForTable(query);
});
query = queryConstructor()
fetchDataForTable(query);
fetchDataForTable();
}
</script>

4
layouts/shortcodes/chart.html
View File

@ -1,10 +1,8 @@
{{ partial "chart.html" }}
<section class = 'chart-container'>
<div class = "chart" id='{{ .Get "id" }}--chart'>
<div id = '{{ .Get "id" }}-chart-container'>
<script>
document.addEventListener("DOMContentLoaded", function () {
createChart(id={{ .Get "id" }}, endpoint={{ .Get "endpoint" }}, chartType={{ .Get "chartType" }}, xAxisField={{ .Get "xAxisField" }}, yAxisField={{ .Get "yAxisField" }}, sortField={{ .Get "sortField" }}, scaleChart={{ .Get "scaleChart" }})
});
</script>
</div>
</section>

27
layouts/shortcodes/dropdown_filter.html
View File

@ -1,27 +0,0 @@
{{ $id := .Get "id" }}
{{ $default_selection := .Get "default_selection" }}
{{ $options := .Get "options" }}
{{ $options_split := split $options "," }}
<div class="dropdown-filter-container">
<select class="filter dropdown-filter" id="{{ $id }}" onchange="dispatchDropdownEvent(this)">
{{ range $options_split }}
{{ $parts := split . ":" }}
{{ $key := index $parts 0 }}
{{ $value := index $parts 1 }}
<option value="{{ $value }}" {{ if eq $key $default_selection }}selected{{ end }}>{{ $key }}</option>
{{ end }}
</select>
<script>
function dispatchDropdownEvent(selectElement) {
const event = new CustomEvent('filterChange', {
detail: {
filterId: selectElement.id,
filterValue: selectElement.value,
}
});
document.dispatchEvent(event);
}
</script>
</div>

2
layouts/shortcodes/table.html
View File

@ -1,5 +1,5 @@
{{ partial "table.html" }}
<div id = '{{ .Get "id" }}--table-container'>
<div id = '{{ .Get "id" }}-table-container'>
<script>
document.addEventListener("DOMContentLoaded", function () {
createTable({{ .Get "endpoint" }}, {{ .Get "id" }}, {{ .Get "headers" | safeJS }}, {{ .Get "maxHeight" }}, {{ .Get "sortable" }}, {{ .Get "valueId" }}, {{ .Get "selectableRows" }})

2
shell.nix
View File

@ -25,7 +25,7 @@ pkgs.mkShell
tmux new-session -d -s baseddata
tmux split-window -h
tmux send-keys -t 0 "hugo server" C-m
tmux send-keys -t 1 "cd backend && uvicorn main:app --reload" C-m
tmux send-keys -t 1 "python backend/main.py" C-m
echo "Baseddata running in dev tmux shell"
fi
'';

25
static/css/charts.css
View File

@ -1,11 +1,24 @@
/* Charts */
.chart-container {
.chart-flex-container {
display: flex;
height: 300px;
}
.chart {
display: flex;
height: 100%;
width: 100%;
.chart-flex-container article {
flex: 1;
}
#chart {
width: 100%;
aspect-ratio: 16 / 9;
}
.chart-container {
margin-top: 20px;
display: flex;
justify-content: center;
flex-direction: column;
}
#chart-modifiers {
display: flex;
}

16
static/js/lib/helper-functions.js
View File

@ -1,16 +0,0 @@
function queryConstructor(customFilters = {}) {
let filters = document.querySelectorAll(".filter");
let queryObject = {};
Object.assign(queryObject, customFilters);
filters.forEach((filter) => {
const filterId = filter.id;
const filterValue = filter.value;
queryObject[filterId] = filterValue;
});
let queryString = `query=${JSON.stringify(queryObject)}`;
return queryString;
}

3
static/js/mangrove-map.js
View File

@ -15,3 +15,6 @@ map.on("load", () => {
});
});
document.addEventListener("DOMContentLoaded", function () {
fetchDataForTable();
});