diff --git a/AGENTS.md b/AGENTS.md index b7c66435..ababedd5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,15 +28,19 @@ docker compose up -d mssql # or postgres or mysql DATABASE_URL='mssql://root:Password123!@localhost/sqlpage' cargo test # all dbms use the same user:pass and db name ``` +### Documentation + +Components and functions are documented in [official website](./examples/official-site/sqlpage/migrations/); one migration per component and per function. You CAN update existing migrations, the official site database is recreated from scratch on each deployment. + +official documentation website sql tables: + - `component(name,description,icon,introduced_in_version)` -- icon name from tabler icon + - `parameter(top_level BOOLEAN, name, component REFERENCES component(name), description, description_md, type, optional BOOLEAN)` parameter types: BOOLEAN, COLOR, HTML, ICON, INTEGER, JSON, REAL, TEXT, TIMESTAMP, URL + - `example(component REFERENCES component(name), description, properties JSON)` + #### Project Conventions - Components: defined in `./sqlpage/templates/*.handlebars` - Functions: `src/webserver/database/sqlpage_functions/functions.rs` registered with `make_function!`. -- Components and functions are documented in [official website](./examples/official-site/sqlpage/migrations/); one migration per component and per function. - - tables - - `component(name,description,icon,introduced_in_version)` -- icon name from tabler icon - - `parameter(top_level BOOLEAN, name, component REFERENCES component(name), description, description_md, type, optional BOOLEAN)` parameter types: BOOLEAN, COLOR, HTML, ICON, INTEGER, JSON, REAL, TEXT, TIMESTAMP, URL - - `example(component REFERENCES component(name), description, properties JSON)` - [Configuration](./configuration.md): see [AppConfig](./src/app_config.rs) - Routing: file-based in `src/webserver/routing.rs`; not found handled via `src/default_404.sql`. - Follow patterns from similar modules before introducing new abstractions. diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c4353a5..7ea53068 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # CHANGELOG.md ## 0.40.0 (unreleased) + - SQLPage now respects [HTTP accept headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Accept) for JSON. You can now easily process the contents of any existing sql page programmatically with: + - `curl -H "Accept: application/json" http://example.com/page.sql`: returns a json array + - `curl -H "Accept: application/x-ndjson" http://example.com/page.sql`: returns one json object per line. - Fixed a bug in `sqlpage.link`: a link with no path (link to the current page) and no url parameter now works as expected. It used to keep the existing url parameters instead of removing them. `sqlpage.link('', '{}')` now returns `'?'` instead of the empty string. - **New Function**: `sqlpage.set_variable(name, value)` - Returns a URL with the specified variable set to the given value, preserving other existing variables. diff --git a/examples/official-site/sqlpage/migrations/11_json.sql b/examples/official-site/sqlpage/migrations/11_json.sql index 65c3a8a1..8384ab58 100644 --- a/examples/official-site/sqlpage/migrations/11_json.sql +++ b/examples/official-site/sqlpage/migrations/11_json.sql @@ -18,6 +18,10 @@ This component **must appear at the top of your SQL file**, before any other dat An HTTP response can have only a single datatype, and it must be declared in the headers. So if you have already called the `shell` component, or another traditional HTML component, you cannot use this component in the same file. + +SQLPage can also return JSON or JSON Lines when the incoming request says it prefers them with an HTTP `Accept` header, so the same `/users.sql` page can show a table in a browser but return raw data to `curl -H "Accept: application/json" http://localhost:8080/users.sql`. + +Use this component when you want to control the payload or force JSON output even for requests that would normally get HTML. ', 'code', '0.9.0' @@ -84,6 +88,12 @@ select * from users; {"username":"James","userid":1}, {"username":"John","userid":2} ] +``` + +Clients can also receive JSON or JSON Lines automatically by requesting the same SQL file with an HTTP `Accept` header such as `application/json` or `application/x-ndjson` when the component is omitted, for example: + +``` +curl -H "Accept: application/json" http://localhost:8080/users.sql ``` ' ), diff --git a/src/render.rs b/src/render.rs index 337d08f5..d2b69857 100644 --- a/src/render.rs +++ b/src/render.rs @@ -42,7 +42,7 @@ //! [SQLPage documentation](https://sql-page.com/documentation.sql). use crate::templates::SplitTemplate; -use crate::webserver::http::RequestContext; +use crate::webserver::http::{RequestContext, ResponseFormat}; use crate::webserver::response_writer::{AsyncResponseWriter, ResponseWriter}; use crate::webserver::ErrorWithStatus; use crate::AppState; @@ -96,11 +96,13 @@ impl HeaderContext { writer: ResponseWriter, ) -> Self { let mut response = HttpResponseBuilder::new(StatusCode::OK); - response.content_type("text/html; charset=utf-8"); - let tpl = &app_state.config.content_security_policy; - request_context - .content_security_policy - .apply_to_response(tpl, &mut response); + response.content_type(request_context.response_format.content_type()); + if request_context.response_format == ResponseFormat::Html { + let tpl = &app_state.config.content_security_policy; + request_context + .content_security_policy + .apply_to_response(tpl, &mut response); + } Self { app_state, request_context, @@ -391,11 +393,23 @@ impl HeaderContext { async fn start_body(mut self, data: JsonValue) -> anyhow::Result { self.add_server_timing_header(); - let html_renderer = - HtmlRenderContext::new(self.app_state, self.request_context, self.writer, data) - .await - .with_context(|| "Failed to create a render context from the header context.")?; - let renderer = AnyRenderBodyContext::Html(html_renderer); + let renderer = match self.request_context.response_format { + ResponseFormat::Json => AnyRenderBodyContext::Json( + JsonBodyRenderer::new_array_with_first_row(self.writer, &data), + ), + ResponseFormat::JsonLines => AnyRenderBodyContext::Json( + JsonBodyRenderer::new_jsonlines_with_first_row(self.writer, &data), + ), + ResponseFormat::Html => { + let html_renderer = + HtmlRenderContext::new(self.app_state, self.request_context, self.writer, data) + .await + .with_context(|| { + "Failed to create a render context from the header context." + })?; + AnyRenderBodyContext::Html(html_renderer) + } + }; let http_response = self.response; Ok(PageContext::Body { renderer, @@ -516,6 +530,11 @@ impl JsonBodyRenderer { let _ = renderer.write_prefix(); renderer } + pub fn new_array_with_first_row(writer: W, first_row: &JsonValue) -> JsonBodyRenderer { + let mut renderer = Self::new_array(writer); + let _ = renderer.handle_row(first_row); + renderer + } pub fn new_jsonlines(writer: W) -> JsonBodyRenderer { let mut renderer = Self { writer, @@ -527,6 +546,11 @@ impl JsonBodyRenderer { renderer.write_prefix().unwrap(); renderer } + pub fn new_jsonlines_with_first_row(writer: W, first_row: &JsonValue) -> JsonBodyRenderer { + let mut renderer = Self::new_jsonlines(writer); + let _ = renderer.handle_row(first_row); + renderer + } pub fn new_server_sent_events(writer: W) -> JsonBodyRenderer { let mut renderer = Self { writer, diff --git a/src/webserver/http.rs b/src/webserver/http.rs index 6c27b593..68464e2b 100644 --- a/src/webserver/http.rs +++ b/src/webserver/http.rs @@ -12,6 +12,7 @@ use crate::webserver::ErrorWithStatus; use crate::{AppConfig, AppState, ParsedSqlFile, DEFAULT_404_FILE}; use actix_web::dev::{fn_service, ServiceFactory, ServiceRequest}; use actix_web::error::{ErrorBadRequest, ErrorInternalServerError}; +use actix_web::http::header::Accept; use actix_web::http::header::{ContentType, Header, HttpDate, IfModifiedSince, LastModified}; use actix_web::http::{header, StatusCode}; use actix_web::web::PayloadConfig; @@ -40,12 +41,52 @@ use std::sync::Arc; use std::time::SystemTime; use tokio::sync::mpsc; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ResponseFormat { + #[default] + Html, + Json, + JsonLines, +} + #[derive(Clone)] pub struct RequestContext { pub is_embedded: bool, pub source_path: PathBuf, pub content_security_policy: ContentSecurityPolicy, pub server_timing: Arc, + pub response_format: ResponseFormat, +} + +impl ResponseFormat { + #[must_use] + pub fn from_accept_header(accept: &Accept) -> Self { + for quality_item in accept.iter() { + let mime = &quality_item.item; + let type_ = mime.type_().as_str(); + let subtype = mime.subtype().as_str(); + + match (type_, subtype) { + ("application", "json") => return Self::Json, + ("application", "x-ndjson" | "jsonlines" | "x-jsonlines") => { + return Self::JsonLines + } + ("text", "x-ndjson" | "jsonlines" | "x-jsonlines") => return Self::JsonLines, + ("text", "html") | ("*", "*") => return Self::Html, + _ => {} + } + } + Self::Html + } + + #[must_use] + pub fn content_type(self) -> &'static str { + match self { + Self::Html => "text/html; charset=utf-8", + Self::Json => "application/json", + Self::JsonLines => "application/x-ndjson", + } + } } async fn stream_response(stream: impl Stream, mut renderer: AnyRenderBodyContext) { @@ -174,6 +215,10 @@ async fn render_sql( .clone() .into_inner(); + let response_format = Accept::parse(srv_req) + .map(|accept| ResponseFormat::from_accept_header(&accept)) + .unwrap_or_default(); + let exec_ctx = extract_request_info(srv_req, Arc::clone(&app_state), server_timing) .await .map_err(|e| anyhow_err_to_actix(e, &app_state))?; @@ -190,6 +235,7 @@ async fn render_sql( source_path, content_security_policy: ContentSecurityPolicy::with_random_nonce(), server_timing: Arc::clone(&request_info.server_timing), + response_format, }; let mut conn = None; let database_entries_stream = diff --git a/tests/data_formats/mod.rs b/tests/data_formats/mod.rs index 56ac1de8..1416f0f0 100644 --- a/tests/data_formats/mod.rs +++ b/tests/data_formats/mod.rs @@ -1,10 +1,23 @@ use actix_web::{ http::{header, StatusCode}, - test, + test::{self, TestRequest}, }; use sqlpage::webserver::http::main_handler; -use crate::common::get_request_to; +use crate::common::{get_request_to, make_app_data}; + +async fn req_with_accept( + path: &str, + accept: &str, +) -> actix_web::Result { + let app_data = make_app_data().await; + let req = TestRequest::get() + .uri(path) + .insert_header((header::ACCEPT, accept)) + .app_data(app_data) + .to_srv_request(); + main_handler(req).await +} #[actix_web::test] async fn test_json_body() -> actix_web::Result<()> { @@ -18,8 +31,7 @@ async fn test_json_body() -> actix_web::Result<()> { resp.headers().get(header::CONTENT_TYPE).unwrap(), "application/json" ); - let body = test::read_body(resp).await; - let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let body_json: serde_json::Value = test::read_body_json(resp).await; assert_eq!( body_json, serde_json::json!([{"message": "It works!"}, {"cool": "cool"}]) @@ -80,3 +92,87 @@ async fn test_json_columns() { "the json should have been parsed, not returned as a string, in: {body_html_escaped}" ); } + +#[actix_web::test] +async fn test_accept_json_returns_json_array() -> actix_web::Result<()> { + let resp = req_with_accept( + "/tests/sql_test_files/it_works_simple.sql", + "application/json", + ) + .await?; + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers().get(header::CONTENT_TYPE).unwrap(), + "application/json" + ); + let body_json: serde_json::Value = test::read_body_json(resp).await; + assert!(body_json.is_array()); + let arr = body_json.as_array().unwrap(); + assert!(arr.len() >= 2); + assert_eq!(arr[0]["component"], "shell"); + assert_eq!(arr[1]["component"], "text"); + Ok(()) +} + +#[actix_web::test] +async fn test_accept_ndjson_returns_jsonlines() -> actix_web::Result<()> { + let resp = req_with_accept( + "/tests/sql_test_files/it_works_simple.sql", + "application/x-ndjson", + ) + .await?; + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers().get(header::CONTENT_TYPE).unwrap(), + "application/x-ndjson" + ); + let body = test::read_body(resp).await; + let body_str = String::from_utf8(body.to_vec()).unwrap(); + let lines: Vec<&str> = body_str.trim().lines().collect(); + assert!(lines.len() >= 2); + assert_eq!( + serde_json::from_str::(lines[0]).unwrap()["component"], + "shell" + ); + assert_eq!( + serde_json::from_str::(lines[1]).unwrap()["component"], + "text" + ); + Ok(()) +} + +#[actix_web::test] +async fn test_accept_html_returns_html() -> actix_web::Result<()> { + let resp = req_with_accept("/tests/sql_test_files/it_works_simple.sql", "text/html").await?; + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers().get(header::CONTENT_TYPE).unwrap(), + "text/html; charset=utf-8" + ); + let body = test::read_body(resp).await; + assert!(body.starts_with(b"")); + Ok(()) +} + +#[actix_web::test] +async fn test_accept_wildcard_returns_html() -> actix_web::Result<()> { + let resp = req_with_accept("/tests/sql_test_files/it_works_simple.sql", "*/*").await?; + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers().get(header::CONTENT_TYPE).unwrap(), + "text/html; charset=utf-8" + ); + Ok(()) +} + +#[actix_web::test] +async fn test_accept_json_redirect_still_works() -> actix_web::Result<()> { + let resp = + req_with_accept("/tests/server_timing/redirect_test.sql", "application/json").await?; + assert_eq!(resp.status(), StatusCode::FOUND); + assert_eq!( + resp.headers().get(header::LOCATION).unwrap(), + "/destination.sql" + ); + Ok(()) +}