Skip to content

Commit 3f900ac

Browse files
ugoaalamb
ugoa
and
alamb
authored
Test all examples from library-user-guide & user-guide docs (apache#14544)
* add mut annotation * fix rust examples * fix rust examples * update * fix first doctest * fix first doctest * fix more doctest * fix more doctest * fix more doctest * adopt rustdoc syntax * adopt rustdoc syntax * adopt rustdoc syntax * fix more doctest * add missing imports * final udtf * reenable * remove dep * run prettier * api-health * update doc * update doc * temp fix * fix doc * fix async schema provider * fix async schema provider * fix doc * fix doc * reorder * refactor * s * finish * minor update * add missing docs * add deps (#3) * fix doctest * update doc * fix doctest * fix doctest * tweak showkeys * fix doctest * fix doctest * fix doctest * fix doctest * update to use user_doc * add rustdoc preprocessing * fix dir * revert to original doc * add allocator * mark type * update * fix doctest * add doctest * add doctest * fix doctest * fix doctest * fix doctest * fix doctest * fix doctest * fix doctest * fix doctest * fix doctest * fix doctest * prettier format * revert change to datafusion-testing * add apache header * install cmake in setup-builder for ci workflow dependency * taplo + fix snmalloc * Update function docs * preprocess user-guide * Render examples as sql * fix intro * fix docs via script --------- Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>
1 parent dee0dc7 commit 3f900ac

23 files changed

Lines changed: 1983 additions & 512 deletions

‎.github/actions/setup-builder/action.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ runs:
3030
run: |
3131
RETRY=("ci/scripts/retry" timeout 120)
3232
"${RETRY[@]}" apt-get update
33-
"${RETRY[@]}" apt-get install -y protobuf-compiler
33+
"${RETRY[@]}" apt-get install -y protobuf-compiler cmake
3434
- name: Setup Rust toolchain
3535
shell: bash
3636
# rustfmt is needed for the substrait build script

‎Cargo.lock‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/core/Cargo.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,13 +95,15 @@ datafusion-catalog = { workspace = true }
9595
datafusion-catalog-listing = { workspace = true }
9696
datafusion-common = { workspace = true, features = ["object_store"] }
9797
datafusion-common-runtime = { workspace = true }
98+
datafusion-doc = { workspace = true }
9899
datafusion-execution = { workspace = true }
99100
datafusion-expr = { workspace = true }
100101
datafusion-functions = { workspace = true }
101102
datafusion-functions-aggregate = { workspace = true }
102103
datafusion-functions-nested = { workspace = true, optional = true }
103104
datafusion-functions-table = { workspace = true }
104105
datafusion-functions-window = { workspace = true }
106+
datafusion-macros = { workspace = true }
105107
datafusion-optimizer = { workspace = true }
106108
datafusion-physical-expr = { workspace = true }
107109
datafusion-physical-expr-common = { workspace = true }
@@ -130,6 +132,7 @@ zstd = { version = "0.13", optional = true, default-features = false }
130132
async-trait = { workspace = true }
131133
criterion = { version = "0.5", features = ["async_tokio"] }
132134
ctor = { workspace = true }
135+
dashmap = "6.1.0"
133136
datafusion-functions-window-common = { workspace = true }
134137
datafusion-physical-optimizer = { workspace = true }
135138
doc-comment = { workspace = true }

‎datafusion/core/src/bin/print_functions_docs.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ fn print_docs(
193193
194194
{}
195195
196-
```
196+
```sql
197197
{}
198198
```
199199
"#,

‎datafusion/core/src/lib.rs‎

Lines changed: 198 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -844,11 +844,17 @@ doc_comment::doctest!("../../../README.md", readme_example_test);
844844
//
845845
// For example, if `user_guide_expressions(line 123)` fails,
846846
// go to `docs/source/user-guide/expressions.md` to find the relevant problem.
847+
//
848+
#[cfg(doctest)]
849+
doc_comment::doctest!(
850+
"../../../docs/source/user-guide/concepts-readings-events.md",
851+
user_guide_concepts_readings_events
852+
);
847853

848854
#[cfg(doctest)]
849855
doc_comment::doctest!(
850-
"../../../docs/source/user-guide/example-usage.md",
851-
user_guide_example_usage
856+
"../../../docs/source/user-guide/configs.md",
857+
user_guide_configs
852858
);
853859

854860
#[cfg(doctest)]
@@ -859,14 +865,20 @@ doc_comment::doctest!(
859865

860866
#[cfg(doctest)]
861867
doc_comment::doctest!(
862-
"../../../docs/source/user-guide/configs.md",
863-
user_guide_configs
868+
"../../../docs/source/user-guide/dataframe.md",
869+
user_guide_dataframe
864870
);
865871

866872
#[cfg(doctest)]
867873
doc_comment::doctest!(
868-
"../../../docs/source/user-guide/dataframe.md",
869-
user_guide_dataframe
874+
"../../../docs/source/user-guide/example-usage.md",
875+
user_guide_example_usage
876+
);
877+
878+
#[cfg(doctest)]
879+
doc_comment::doctest!(
880+
"../../../docs/source/user-guide/explain-usage.md",
881+
user_guide_explain_usage
870882
);
871883

872884
#[cfg(doctest)]
@@ -875,20 +887,197 @@ doc_comment::doctest!(
875887
user_guide_expressions
876888
);
877889

890+
#[cfg(doctest)]
891+
doc_comment::doctest!("../../../docs/source/user-guide/faq.md", user_guide_faq);
892+
878893
#[cfg(doctest)]
879894
doc_comment::doctest!(
880-
"../../../docs/source/library-user-guide/using-the-sql-api.md",
881-
library_user_guide_sql_api
895+
"../../../docs/source/user-guide/introduction.md",
896+
user_guide_introduction
897+
);
898+
899+
#[cfg(doctest)]
900+
doc_comment::doctest!(
901+
"../../../docs/source/user-guide/cli/datasources.md",
902+
user_guide_cli_datasource
903+
);
904+
905+
#[cfg(doctest)]
906+
doc_comment::doctest!(
907+
"../../../docs/source/user-guide/cli/installation.md",
908+
user_guide_cli_installation
909+
);
910+
911+
#[cfg(doctest)]
912+
doc_comment::doctest!(
913+
"../../../docs/source/user-guide/cli/overview.md",
914+
user_guide_cli_overview
915+
);
916+
917+
#[cfg(doctest)]
918+
doc_comment::doctest!(
919+
"../../../docs/source/user-guide/cli/usage.md",
920+
user_guide_cli_usage
921+
);
922+
923+
#[cfg(doctest)]
924+
doc_comment::doctest!(
925+
"../../../docs/source/user-guide/sql/aggregate_functions.md",
926+
user_guide_sql_aggregate_functions
927+
);
928+
929+
#[cfg(doctest)]
930+
doc_comment::doctest!(
931+
"../../../docs/source/user-guide/sql/data_types.md",
932+
user_guide_sql_data_types
933+
);
934+
935+
#[cfg(doctest)]
936+
doc_comment::doctest!(
937+
"../../../docs/source/user-guide/sql/ddl.md",
938+
user_guide_sql_ddl
939+
);
940+
941+
#[cfg(doctest)]
942+
doc_comment::doctest!(
943+
"../../../docs/source/user-guide/sql/dml.md",
944+
user_guide_sql_dml
945+
);
946+
947+
#[cfg(doctest)]
948+
doc_comment::doctest!(
949+
"../../../docs/source/user-guide/sql/explain.md",
950+
user_guide_sql_exmplain
951+
);
952+
953+
#[cfg(doctest)]
954+
doc_comment::doctest!(
955+
"../../../docs/source/user-guide/sql/information_schema.md",
956+
user_guide_sql_information_schema
957+
);
958+
959+
#[cfg(doctest)]
960+
doc_comment::doctest!(
961+
"../../../docs/source/user-guide/sql/operators.md",
962+
user_guide_sql_operators
963+
);
964+
965+
#[cfg(doctest)]
966+
doc_comment::doctest!(
967+
"../../../docs/source/user-guide/sql/scalar_functions.md",
968+
user_guide_sql_scalar_functions
969+
);
970+
971+
#[cfg(doctest)]
972+
doc_comment::doctest!(
973+
"../../../docs/source/user-guide/sql/select.md",
974+
user_guide_sql_select
975+
);
976+
977+
#[cfg(doctest)]
978+
doc_comment::doctest!(
979+
"../../../docs/source/user-guide/sql/special_functions.md",
980+
user_guide_sql_special_functions
981+
);
982+
983+
#[cfg(doctest)]
984+
doc_comment::doctest!(
985+
"../../../docs/source/user-guide/sql/sql_status.md",
986+
user_guide_sql_status
987+
);
988+
989+
#[cfg(doctest)]
990+
doc_comment::doctest!(
991+
"../../../docs/source/user-guide/sql/subqueries.md",
992+
user_guide_sql_subqueries
993+
);
994+
995+
#[cfg(doctest)]
996+
doc_comment::doctest!(
997+
"../../../docs/source/user-guide/sql/window_functions.md",
998+
user_guide_sql_window_functions
999+
);
1000+
1001+
#[cfg(doctest)]
1002+
doc_comment::doctest!(
1003+
"../../../docs/source/user-guide/sql/write_options.md",
1004+
user_guide_sql_write_options
1005+
);
1006+
1007+
#[cfg(doctest)]
1008+
doc_comment::doctest!(
1009+
"../../../docs/source/library-user-guide/adding-udfs.md",
1010+
library_user_guide_adding_udfs
1011+
);
1012+
1013+
#[cfg(doctest)]
1014+
doc_comment::doctest!(
1015+
"../../../docs/source/library-user-guide/api-health.md",
1016+
library_user_guide_api_health
8821017
);
8831018

8841019
#[cfg(doctest)]
8851020
doc_comment::doctest!(
8861021
"../../../docs/source/library-user-guide/building-logical-plans.md",
887-
library_user_guide_logical_plans
1022+
library_user_guide_building_logical_plans
1023+
);
1024+
1025+
#[cfg(doctest)]
1026+
doc_comment::doctest!(
1027+
"../../../docs/source/library-user-guide/catalogs.md",
1028+
library_user_guide_catalogs
1029+
);
1030+
1031+
#[cfg(doctest)]
1032+
doc_comment::doctest!(
1033+
"../../../docs/source/library-user-guide/custom-table-providers.md",
1034+
library_user_guide_custom_table_providers
1035+
);
1036+
1037+
#[cfg(doctest)]
1038+
doc_comment::doctest!(
1039+
"../../../docs/source/library-user-guide/extending-operators.md",
1040+
library_user_guide_extending_operators
1041+
);
1042+
1043+
#[cfg(doctest)]
1044+
doc_comment::doctest!(
1045+
"../../../docs/source/library-user-guide/extensions.md",
1046+
library_user_guide_extensions
1047+
);
1048+
1049+
#[cfg(doctest)]
1050+
doc_comment::doctest!(
1051+
"../../../docs/source/library-user-guide/index.md",
1052+
library_user_guide_index
1053+
);
1054+
1055+
#[cfg(doctest)]
1056+
doc_comment::doctest!(
1057+
"../../../docs/source/library-user-guide/profiling.md",
1058+
library_user_guide_profiling
1059+
);
1060+
1061+
#[cfg(doctest)]
1062+
doc_comment::doctest!(
1063+
"../../../docs/source/library-user-guide/query-optimizer.md",
1064+
library_user_guide_query_optimizer
8881065
);
8891066

8901067
#[cfg(doctest)]
8911068
doc_comment::doctest!(
8921069
"../../../docs/source/library-user-guide/using-the-dataframe-api.md",
8931070
library_user_guide_dataframe_api
8941071
);
1072+
1073+
#[cfg(doctest)]
1074+
doc_comment::doctest!(
1075+
"../../../docs/source/library-user-guide/using-the-sql-api.md",
1076+
library_user_guide_sql_api
1077+
);
1078+
1079+
#[cfg(doctest)]
1080+
doc_comment::doctest!(
1081+
"../../../docs/source/library-user-guide/working-with-exprs.md",
1082+
library_user_guide_working_with_exprs
1083+
);

‎dev/update_function_docs.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ WINDOW w AS (PARTITION BY depname ORDER BY salary DESC);
236236
237237
The syntax for the OVER-clause is
238238
239-
```
239+
```sql
240240
function([expr])
241241
OVER(
242242
[PARTITION BY expr[, …]]
@@ -247,7 +247,7 @@ function([expr])
247247
248248
where **frame_clause** is one of:
249249
250-
```
250+
```sql
251251
{ RANGE | ROWS | GROUPS } frame_start
252252
{ RANGE | ROWS | GROUPS } BETWEEN frame_start AND frame_end
253253
```

‎docs/build.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,7 @@ mkdir temp
2525
cp -rf source/* temp/
2626
# replace relative URLs with absolute URLs
2727
sed -i -e 's/\.\.\/\.\.\/\.\.\//https:\/\/github.com\/apache\/arrow-datafusion\/blob\/main\//g' temp/contributor-guide/index.md
28+
29+
python rustdoc_trim.py
30+
2831
make SOURCEDIR=`pwd`/temp html

0 commit comments

Comments
 (0)
⚡