{ "_id": "6a106b69acfb0bcc41caa4a8", "Package": "OptHoldoutSize", "Title": "Estimation of Optimal Size for a Holdout Set for Updating a\nPredictive Score", "Version": "0.1.0.2", "Authors@R": "c(person(given = \"Sami\",\nfamily = \"Haidar-Wehbe\",\nrole = c(\"aut\"),\nemail = \"sami.haidar96@gmail.com\"),\nperson(given = \"Sam\",\nfamily = \"Emerson\",\nrole = c(\"aut\"),\nemail = \"samuel.r.emerson@durham.ac.uk\",\ncomment = c(ORCID = \"0000-0002-8379-2781\")),\nperson(given = \"Louis\",\nfamily = \"Aslett\",\nrole = c(\"aut\"),\nemail = \"louis.aslett@durham.ac.uk\",\ncomment = c(ORCID = \"0000-0003-2211-233X\")),\nperson(given = \"James\",\nfamily = \"Liley\",\nrole = c(\"cre\",\"aut\"),\nemail = \"james.liley@durham.ac.uk\",\ncomment = c(ORCID = \"0000-0002-0049-8238\")))", "Maintainer": "James Liley ", "Description": "Predictive scores must be updated with care, because\nactions taken on the basis of existing risk scores causes bias\nin risk estimates from the updated score. A holdout set is a\nstraightforward way to manage this problem: a proportion of the\npopulation is 'held-out' from computation of the previous risk\nscore. This package provides tools to estimate a size for this\nholdout set and associated errors. Comprehensive vignettes are\nincluded. Please see: Haidar-Wehbe S, Emerson SR, Aslett LJM,\nLiley J (2022) (in Annals of\nApplied Statistics) for details of methods.", "License": "GPL (>= 3)", "Encoding": "UTF-8", "LazyData": "true", "VignetteBuilder": "knitr", "RoxygenNote": "7.3.2", "NeedsCompilation": "no", "Packaged": { "Date": "2026-05-07 09:24:57 UTC", "User": "root" }, "Author": "Sami Haidar-Wehbe [aut], Sam Emerson [aut] (ORCID:\n), Louis Aslett [aut]\n(ORCID: ), James Liley\n[cre, aut] (ORCID: )", "Repository": "https://jamesliley.r-universe.dev", "Date/Publication": "2026-04-07 11:17:42 UTC", "RemoteUrl": "https://github.com/cran/OptHoldoutSize", "RemoteRef": "HEAD", "RemoteSha": "a1cf00a496f58c79f24d11fb14976e5f1b160ab6", "MD5sum": "44febb03ee7ceb5a60c3f6a1e9817f08", "_user": "jamesliley", "_type": "src", "_file": "OptHoldoutSize_0.1.0.2.tar.gz", "_fileid": "10102e18cfd4ebf0aa919ce89c5f6610eb41061fc8c494fba654a9e13dbed000", "_filesize": 4381775, "_sha256": "10102e18cfd4ebf0aa919ce89c5f6610eb41061fc8c494fba654a9e13dbed000", "_created": "2026-05-07T09:24:57.000Z", "_published": "2026-05-22T14:42:49.699Z", "_distro": "noble", "_jobs": [ { "job": 77402561645, "time": 173, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "WARNING", "artifact": "6851553383" }, { "job": 77402562588, "time": 165, "config": "linux-release-x86_64", "r": "4.6.0", "check": "WARNING", "artifact": "6851548382" }, { "job": 77402562104, "time": 149, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "WARNING", "artifact": "6851536633" }, { "job": 77402561785, "time": 146, "config": "macos-release-arm64", "r": "4.6.0", "check": "WARNING", "artifact": "6851536146" }, { "job": 77402560430, "time": 161, "config": "source", "r": "4.6.0", "check": "OK", "artifact": "6851491913" }, { "job": 77402560916, "time": 105, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "7163265238" }, { "job": 77402561264, "time": 117, "config": "windows-devel", "r": "4.7.0", "check": "WARNING", "artifact": "6851532935" }, { "job": 77402561690, "time": 98, "config": "windows-oldrel", "r": "4.5.3", "check": "WARNING", "artifact": "6851525909" }, { "job": 77402562431, "time": 90, "config": "windows-release", "r": "4.6.0", "check": "WARNING", "artifact": "6851523507" } ], "_buildurl": "https://github.com/r-universe/jamesliley/actions/runs/25487315478", "_status": "success", "_host": "GitHub-Actions", "_upstream": "https://github.com/cran/OptHoldoutSize", "_commit": { "id": "a1cf00a496f58c79f24d11fb14976e5f1b160ab6", "author": "James Liley ", "committer": "cran-robot ", "message": "version 0.1.0.2\n", "time": 1775560662 }, "_maintainer": { "name": "James Liley", "email": "james.liley@durham.ac.uk", "login": "jamesliley", "uuid": 7887561, "orcid": "0000-0002-0049-8238" }, "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 3.5.0", "role": "Depends" }, { "package": "matrixStats", "role": "Depends" }, { "package": "mnormt", "role": "Depends" }, { "package": "mvtnorm", "role": "Depends" }, { "package": "ranger", "role": "Depends" }, { "package": "knitr", "role": "Suggests" }, { "package": "rmarkdown", "role": "Suggests" } ], "_owner": "cran", "_selfowned": true, "_usedby": 0, "_updates": [ { "week": "2026-15", "n": 1 } ], "_tags": [ { "name": "0.1.0.2", "date": "2026-04-07" } ], "_stars": 0, "_contributors": [ { "user": "jamesliley", "count": 4, "uuid": 7887561 } ], "_userbio": { "uuid": 7887561, "type": "user", "name": "jamesliley" }, "_downloads": { "count": 502, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/OptHoldoutSize" }, "_searchresults": 9, "_rbuild": "4.6.0", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/NEWS.html", "extra/NEWS.txt", "extra/OptHoldoutSize.html", "extra/readme.html", "extra/readme.md", "manual.pdf" ], "_realowner": "jamesliley", "_cranurl": false, "_releases": [ { "version": "0.0.0.1", "date": "2022-02-09" }, { "version": "0.1.0.0", "date": "2022-02-18" }, { "version": "0.1.0.1", "date": "2025-04-09" }, { "version": "0.1.0.2", "date": "2026-04-07" } ], "_exports": [ "add_aspre_interactions", "aspre", "aspre_k2", "ci_mincost", "ci_ohs", "cov_fn", "error_ohs_emulation", "exp_imp_fn", "gen_base_coefs", "gen_preds", "gen_resp", "grad_mincost_powerlaw", "grad_nstar_powerlaw", "logistic", "logit", "model_predict", "model_train", "mu_fn", "next_n", "optimal_holdout_size", "optimal_holdout_size_emulation", "oracle_pred", "powerlaw", "powersolve", "powersolve_general", "powersolve_se", "psi_fn", "sens10", "sim_random_aspre", "split_data" ], "_datasets": [ { "name": "aspre_emulation", "title": "Emulation-based OHS estimation for ASPRE", "object": "aspre_emulation", "class": [ "list" ], "fields": [], "table": true, "tojson": true }, { "name": "aspre_parametric", "title": "Parametric-based OHS estimation for ASPRE", "object": "aspre_parametric", "class": [ "list" ], "fields": [], "table": true, "tojson": true }, { "name": "ci_cover_a_yn", "title": "Data for example on asymptotic confidence interval for OHS.", "object": "ci_cover_a_yn", "class": [ "matrix", "array" ], "fields": {}, "rows": 11, "table": true, "tojson": true }, { "name": "ci_cover_cost_a_yn", "title": "Data for example on asymptotic confidence interval for min cost.", "object": "ci_cover_cost_a_yn", "class": [ "matrix", "array" ], "fields": {}, "rows": 11, "table": true, "tojson": true }, { "name": "ci_cover_cost_e_yn", "title": "Data for example on empirical confidence interval for min cost.", "object": "ci_cover_cost_e_yn", "class": [ "matrix", "array" ], "fields": {}, "rows": 11, "table": true, "tojson": true }, { "name": "ci_cover_e_yn", "title": "Data for example on empirical confidence interval for OHS.", "object": "ci_cover_e_yn", "class": [ "matrix", "array" ], "fields": {}, "rows": 11, "table": true, "tojson": true }, { "name": "data_example_simulation", "title": "Data for vignette showing general example", "object": "data_example_simulation", "class": [ "list" ], "fields": [], "table": true, "tojson": true }, { "name": "data_nextpoint_em", "title": "Data for 'next point' demonstration vignette on algorithm comparison using emulation algorithm", "object": "data_nextpoint_em", "class": [ "list" ], "fields": [], "table": true, "tojson": true }, { "name": "data_nextpoint_par", "title": "Data for 'next point' demonstration vignette on algorithm comparison using parametric algorithm", "object": "data_nextpoint_par", "class": [ "list" ], "fields": [], "table": true, "tojson": true }, { "name": "ohs_array", "title": "Data for vignette on algorithm comparison", "object": "ohs_array", "class": [ "array" ], "fields": [], "table": false, "tojson": true }, { "name": "ohs_resample", "title": "Data for vignette on algorithm comparison", "object": "ohs_resample", "class": [ "matrix", "array" ], "fields": [ "param_pTRUE", "param_pFALSE", "emul_pTRUE", "emul_pFALSE" ], "rows": 1000, "table": true, "tojson": true }, { "name": "params_aspre", "title": "Parameters of reported ASPRE dataset", "object": "params_aspre", "class": [ "list" ], "fields": [], "table": false, "tojson": true } ], "_help": [ { "page": "add_aspre_interactions", "title": "Add interaction terms corresponding to ASPRE model", "topics": [ "add_aspre_interactions" ] }, { "page": "aspre", "title": "Computes ASPRE score", "topics": [ "aspre" ] }, { "page": "aspre_emulation", "title": "Emulation-based OHS estimation for ASPRE", "topics": [ "aspre_emulation" ] }, { "page": "aspre_k2", "title": "Cost estimating function in ASPRE simulation", "topics": [ "aspre_k2" ] }, { "page": "aspre_parametric", "title": "Parametric-based OHS estimation for ASPRE", "topics": [ "aspre_parametric" ] }, { "page": "ci_cover_a_yn", "title": "Data for example on asymptotic confidence interval for OHS.", "topics": [ "ci_cover_a_yn" ] }, { "page": "ci_cover_cost_a_yn", "title": "Data for example on asymptotic confidence interval for min cost.", "topics": [ "ci_cover_cost_a_yn" ] }, { "page": "ci_cover_cost_e_yn", "title": "Data for example on empirical confidence interval for min cost.", "topics": [ "ci_cover_cost_e_yn" ] }, { "page": "ci_cover_e_yn", "title": "Data for example on empirical confidence interval for OHS.", "topics": [ "ci_cover_e_yn" ] }, { "page": "ci_mincost", "title": "Confidence interval for minimum total cost, when estimated using parametric method", "topics": [ "ci_mincost" ] }, { "page": "ci_ohs", "title": "Confidence interval for optimal holdout size, when estimated using parametric method", "topics": [ "ci_ohs" ] }, { "page": "cov_fn", "title": "Covariance function for Gaussian process", "topics": [ "cov_fn" ] }, { "page": "data_example_simulation", "title": "Data for vignette showing general example", "topics": [ "data_example_simulation" ] }, { "page": "data_nextpoint_em", "title": "Data for 'next point' demonstration vignette on algorithm comparison using emulation algorithm", "topics": [ "data_nextpoint_em" ] }, { "page": "data_nextpoint_par", "title": "Data for 'next point' demonstration vignette on algorithm comparison using parametric algorithm", "topics": [ "data_nextpoint_par" ] }, { "page": "error_ohs_emulation", "title": "Measure of error for emulation-based OHS emulation", "topics": [ "error_ohs_emulation" ] }, { "page": "exp_imp_fn", "title": "Expected improvement", "topics": [ "exp_imp_fn" ] }, { "page": "gen_base_coefs", "title": "Coefficients for imperfect risk score", "topics": [ "gen_base_coefs" ] }, { "page": "gen_preds", "title": "Generate matrix of random observations", "topics": [ "gen_preds" ] }, { "page": "gen_resp", "title": "Generate response", "topics": [ "gen_resp" ] }, { "page": "grad_mincost_powerlaw", "title": "Gradient of minimum cost (power law)", "topics": [ "grad_mincost_powerlaw" ] }, { "page": "grad_nstar_powerlaw", "title": "Gradient of optimal holdout size (power law)", "topics": [ "grad_nstar_powerlaw" ] }, { "page": "logistic", "title": "Logistic", "topics": [ "logistic" ] }, { "page": "logit", "title": "Logit", "topics": [ "logit" ] }, { "page": "model_predict", "title": "Make predictions", "topics": [ "model_predict" ] }, { "page": "model_train", "title": "Train model (wrapper)", "topics": [ "model_train" ] }, { "page": "mu_fn", "title": "Updating function for mean.", "topics": [ "mu_fn" ] }, { "page": "next_n", "title": "Finds best value of n to sample next", "topics": [ "next_n" ] }, { "page": "ohs_array", "title": "Data for vignette on algorithm comparison", "topics": [ "ohs_array" ] }, { "page": "ohs_resample", "title": "Data for vignette on algorithm comparison", "topics": [ "ohs_resample" ] }, { "page": "optimal_holdout_size", "title": "Estimate optimal holdout size under parametric assumptions", "topics": [ "optimal_holdout_size" ] }, { "page": "optimal_holdout_size_emulation", "title": "Estimate optimal holdout size under semi-parametric assumptions", "topics": [ "optimal_holdout_size_emulation" ] }, { "page": "oracle_pred", "title": "Generate responses", "topics": [ "oracle_pred" ] }, { "page": "params_aspre", "title": "Parameters of reported ASPRE dataset", "topics": [ "params_aspre" ] }, { "page": "plot.optholdoutsize", "title": "Plot estimated cost function", "topics": [ "plot.optholdoutsize" ] }, { "page": "plot.optholdoutsize_emul", "title": "Plot estimated cost function using emulation (semiparametric)", "topics": [ "plot.optholdoutsize_emul" ] }, { "page": "powerlaw", "title": "Power law function", "topics": [ "powerlaw" ] }, { "page": "powersolve", "title": "Fit power law curve", "topics": [ "powersolve" ] }, { "page": "powersolve_general", "title": "General solver for power law curve", "topics": [ "powersolve_general" ] }, { "page": "powersolve_se", "title": "Standard error matrix for learning curve parameters (power law)", "topics": [ "powersolve_se" ] }, { "page": "psi_fn", "title": "Updating function for variance.", "topics": [ "psi_fn" ] }, { "page": "sens10", "title": "Sensitivity at theshold quantile 10%", "topics": [ "sens10" ] }, { "page": "sim_random_aspre", "title": "Simulate random dataset similar to ASPRE training data", "topics": [ "sim_random_aspre" ] }, { "page": "split_data", "title": "Split data", "topics": [ "split_data" ] } ], "_readme": "https://github.com/cran/OptHoldoutSize/raw/HEAD/README.md", "_rundeps": [ "lattice", "Matrix", "matrixStats", "mnormt", "mvtnorm", "ranger", "Rcpp", "RcppEigen" ], "_vignettes": [ { "source": "ASPRE_example.Rmd", "filename": "ASPRE_example.html", "title": "ASPRE example", "author": "Sami Haidar-Wehbe, Sam Emerson, Louis Aslett, James Liley", "engine": "knitr::rmarkdown", "headings": [ "Introduction", "Estimation of optimal holdout size", "Parameters $N$ and $k_1$", "Simulation of dataset", "Computation of OHS using parametric method", "Start with estimates of k2 at 10 values of n", "Candidate values for n", "Starting value for theta", "Rough estimate for variance of k2", "Successively add new points", "Resample k2(n), to avoid double-dipping effect", "Transform to total cost", "Save", "Begin as for parametric approach", "Transform estimated k2 to costs" ], "created": "2022-02-09 08:50:08", "modified": "2026-04-07 11:17:42", "commits": 3 }, { "source": "comparison_of_algorithms.Rmd", "filename": "comparison_of_algorithms.html", "title": "Comparison of algorithms", "author": "Sami Haidar-Wehbe, Sam Emerson, Louis Aslett, James Liley", "engine": "knitr::rmarkdown", "headings": [ "Introduction", "Setup", "Assumptions throughout", "General setup", "Demonstration of algorithms", "Parametric assumptions", "Simulate data", "Resample values d", "Estimate a,b, and c from values nset and d", "Estimate optimal holdout sizes using parametric method", "Estimate optimal holdout sizes using semi-parametric (emulation) method", "To draw plot:", "These are our sets of training sizes and k2 estimates, which will be built up.", "Go up to this many points", "Estimate parameters", "Find next suggested point, parametric assumptions satisfied", "Find next suggested point, parametric assumptions not satisfied", "New estimates of k2", "Update data", "Sys.sleep(10)", "To draw plot with np points (np can be set using the button)", "Estimate parameters for parametric part of semi-parametric method", "First panel", "Second panel", "Rates of convergence", "Load datasets of 'next point'", "Maximum number of training set sizes we will consider", "Generate random 'next points'", "Initialise matrices of records", "ohs_array[n,i,j,k,l] is", "using the first n training set sizes", "the ith resample", "using the jth version of k2 (j=1: pTRUE, j=2: pFALSE)", "using the kth algorithm (k=1: parametric, k=2: semiparametric/emulation)", "using the lth method of selecting next points (l=1: random, l=2: systematic)", "Resamplings for parametric algorithm, random next point", "Resamplings for semiparametric/emulation algorithm, random next point", "Resamplings for parametric algorithm, nonrandom (systematic) next point", "Resamplings for semiparametric/emulation algorithm, nonrandom (systematic) next point", "Load data", "Settings", "Plot drawing function", "Set up plot parameters", "Number of estimates", "Initialise", "Plot medians", "Ranges", "Coarsening factor: coarsen OHS estimates to nearest value", "Record lengths of rounded OHS numbers", "Plot ranges", "Add legend", "Bottom panel setup", "Root mean square errors", "Draw lines", "Extract matrices from aray", "True OHS", "True functions l" ], "created": "2022-02-09 08:50:08", "modified": "2025-04-09 11:30:06", "commits": 3 }, { "source": "simulated_example.Rmd", "filename": "simulated_example.html", "title": "Simulated example", "author": "Sami Haidar-Wehbe, Sam Emerson, Louis Aslett, James Liley", "engine": "knitr::rmarkdown", "headings": [ "Introduction", "Setup", "Assumptions", "General setup", "Results", "Emergence of optimal holdout size", "Cost of true negative, false positive etc for computing total cost", "Generate coefficients for underlying model and for predictions in h.o. set", "Set ground truth coefficients, and the accuracy at baseline", "Run simulation", "If ninters>0, append coefficients for interaction terms to coefficients", "Collate and save all data", "Load data", "Holdout set size in absolute terms", "Colour intensities", "Draw cost function", "Cost values", "Initialise plot for cost function", "Plot Cost line", "Standard Deviation Bands", "Add legend", "Draw k2 function", "Evaluate k2(n)*(N-n)", "Evaluate k2", "Scalinge factor (for plotting)", "Initialise plot for k2 function", "Line for estimated k2(n)", "Add standard deviation of k2(n).", "Line for estimated k2(n)*(N-n)", "Add standard deviation for k2(n)*(N-n)", "Draw k1*n function", "Evaluate k1", "Line for estimated k1*n", "Add standard deviation. Note this is scaled by (nobs-n_ho)", "Loop through values of 'families' and 'interactions'" ], "created": "2022-02-09 08:50:08", "modified": "2025-04-09 11:30:06", "commits": 2 } ], "_score": 3.1760912590556813, "_indexed": true, "_nocasepkg": "optholdoutsize", "_universes": [ "jamesliley" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "0.1.0.2", "date": "2026-05-07T09:27:40.000Z", "distro": "noble", "commit": "a1cf00a496f58c79f24d11fb14976e5f1b160ab6", "fileid": "2686f9583316eef325d4f21eb27073b63fc59a6686ae6dd1036eaf03a9927abb", "status": "success", "check": "WARNING", "buildurl": "https://github.com/r-universe/jamesliley/actions/runs/25487315478" }, { "r": "4.6.0", "os": "linux", "version": "0.1.0.2", "date": "2026-05-07T09:27:29.000Z", "distro": "noble", "commit": "a1cf00a496f58c79f24d11fb14976e5f1b160ab6", "fileid": "1352c83cee46ef676e6b0cf9ecfdb39941c6ac14e8a4904bcd79f2f1adcce811", "status": "success", "check": "WARNING", "buildurl": "https://github.com/r-universe/jamesliley/actions/runs/25487315478" }, { "r": "4.5.3", "os": "mac", "version": "0.1.0.2", "date": "2026-05-07T09:27:07.000Z", "commit": "a1cf00a496f58c79f24d11fb14976e5f1b160ab6", "fileid": "4da49603fcb59ede6454849b58d1e3fcf24ef8b80e6dd52a8ee68b6662d8f059", "status": "success", "check": "WARNING", "buildurl": "https://github.com/r-universe/jamesliley/actions/runs/25487315478" }, { "r": "4.6.0", "os": "mac", "version": "0.1.0.2", "date": "2026-05-07T09:27:06.000Z", "commit": "a1cf00a496f58c79f24d11fb14976e5f1b160ab6", "fileid": "d6ad2773c3f10fe32874b7d2952e9710b5357c1f3f10dcfb89a16485fa28633d", "status": "success", "check": "WARNING", "buildurl": "https://github.com/r-universe/jamesliley/actions/runs/25487315478" }, { "r": "4.7.0", "os": "win", "version": "0.1.0.2", "date": "2026-05-07T09:26:32.000Z", "commit": "a1cf00a496f58c79f24d11fb14976e5f1b160ab6", "fileid": "8293194cb1b8cf6df00f7edffa3f2f929b06bf30133eb279688d10fc76798662", "status": "success", "check": "WARNING", "buildurl": "https://github.com/r-universe/jamesliley/actions/runs/25487315478" }, { "r": "4.5.3", "os": "win", "version": "0.1.0.2", "date": "2026-05-07T09:26:10.000Z", "commit": "a1cf00a496f58c79f24d11fb14976e5f1b160ab6", "fileid": "9b6749d5e1bb834f31bf0397d9997aad7a081c55eb5750607ba7682ea8950d84", "status": "success", "check": "WARNING", "buildurl": "https://github.com/r-universe/jamesliley/actions/runs/25487315478" }, { "r": "4.6.0", "os": "win", "version": "0.1.0.2", "date": "2026-05-07T09:26:07.000Z", "commit": "a1cf00a496f58c79f24d11fb14976e5f1b160ab6", "fileid": "2defab5ee2abc74fa678403c0ec89e5a193ae7f3e51b0f6730e73603fad304b4", "status": "success", "check": "WARNING", "buildurl": "https://github.com/r-universe/jamesliley/actions/runs/25487315478" }, { "r": "4.6.0", "os": "wasm", "version": "0.1.0.2", "date": "2026-05-22T14:42:28.000Z", "commit": "a1cf00a496f58c79f24d11fb14976e5f1b160ab6", "fileid": "3bae9d5de0e91b9caf31824c1d6a691d4131644c1fa2a10841a520fe4f2c76a2", "status": "success", "buildurl": "https://github.com/r-universe/jamesliley/actions/runs/25487315478" } ] }