{ "_id": "6acb5422fe5cc25786e8e3c2", "Package": "Gtheory4LLM", "Type": "Package", "Title": "Generalizability Theory for LLM Subjective Tasks", "Version": "0.5.0", "Authors@R": "person(\"Jin\", \"Liu\", email = \"Veronica.Liu0206@gmail.com\",\nrole = c(\"aut\", \"cre\", \"cph\"))", "Author": "Jin Liu [aut, cre, cph]", "Maintainer": "Jin Liu ", "Description": "Studies the reliability and generalizability of subjective\njudgments produced by large language models (LLMs), including\nannotation, rating, and structured LLM-as-a-judge tasks.\nSpecifies evaluator, prompt, generation, and repeated-run\nfacets through crossed or explicitly nested random sources with\nconfigurable item interactions. Fits univariate models, joint\nGaussian models, and joint discrete models for binary, ordinal,\nand unordered categorical outcomes, with source-specific\ncovariance. Gaussian models use exact balanced likelihood;\ndiscrete models use a dense first-order Laplace approximation\nwith Gaussian latent random effects. Supported balanced\ndecision studies compare evaluator, prompt, and replication\nallocations using observed Gaussian or explicitly requested\nlatent binary and ordinal reliability, with random or fixed\nfacets after Brennan (2001). Gaussian fits report asymptotic\nWald standard errors for their variance components and\ndelta-method intervals for the coefficients; discrete fits\nreport point estimates only. Provides fixed-layout Gaussian\nbatch projections, bounded cost-aware allocation comparisons,\nand Gaussian simulation-based pilot precision planning. Scalar\nnominal reliability and joint Gaussian-discrete fitting are not\nimplemented. Includes three publicly archived LLM annotation\ndatasets covering hate-speech, mental-health, and drug-review\ntasks. Discrete fitting is limited to small models; the\npreflight report describes supported designs and computational\nlimits. Generalizability coefficients follow the\nvariance-decomposition framework of Brennan (2001)\n.", "License": "GPL-3", "Encoding": "UTF-8", "VignetteBuilder": "knitr", "URL": "https://github.com/Veronica0206/Gtheory4LLM", "BugReports": "https://github.com/Veronica0206/Gtheory4LLM/issues", "Collate": "'design.R' 'batch.R' 'family.R' 'gaussian_retry.R'\n'gaussian_engine.R' 'gaussian.R' 'discrete_response.R'\n'discrete_dense.R' 'discrete_sparse.R' 'discrete_mode.R'\n'discrete_sparse_mode.R' 'discrete.R' 'preflight.R'\n'diagnostics_stages.R' 'fit.R' 'methods.R' 'reliability.R'\n'batch_reliability.R' 'planning.R' 'simulation.R'\n'visualization.R' 'report.R' 'examples.R'", "Config/pak/sysreqs": "cmake make", "Repository": "https://veronica0206.r-universe.dev", "Date/Publication": "2026-10-11 02:26:39 UTC", "RemoteUrl": "https://github.com/veronica0206/gtheory4llm", "RemoteRef": "HEAD", "RemoteSha": "e45cfa40dad8b99a17b77cea5397d61fb64f3c90", "NeedsCompilation": "no", "Packaged": { "Date": "2026-10-11 09:08:46 UTC", "User": "root" }, "_user": "veronica0206", "_type": "src", "_file": "Gtheory4LLM_0.5.0.tar.gz", "_fileid": "https://r2.ropensci.org/6c436cac980c3060f1419c2464e4354653d623b24937f13c11e458a19c893623", "_filesize": 1071340, "_sha256": "6c436cac980c3060f1419c2464e4354653d623b24937f13c11e458a19c893623", "_expires": "2027-01-19T09:17:20.000Z", "_created": "2026-10-11T09:08:46.000Z", "_published": "2026-10-11T09:17:22.251Z", "_buildurl": "https://github.com/r-universe/veronica0206/actions/runs/38127088638", "_status": "success", "_host": "GitHub-Actions", "_jobs": [ { "job": 114434364490, "time": 237, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "11697638912" }, { "job": 114434364438, "time": 302, "config": "linux-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "11697133960" }, { "job": 114434364525, "time": 240, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "OK", "artifact": "11697064275" }, { "job": 114434364432, "time": 299, "config": "macos-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "11697718922" }, { "job": 114433649349, "time": 241, "config": "source", "r": "4.6.1", "check": "OK", "artifact": "11697068639" }, { "job": 114434364439, "time": 112, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "11697319167" }, { "job": 114434364480, "time": 325, "config": "windows-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "11696894339" }, { "job": 114434364494, "time": 333, "config": "windows-oldrel-x86_64", "r": "4.5.3", "check": "OK", "artifact": "11697908300" }, { "job": 114434364551, "time": 217, "config": "windows-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "11697498973" } ], "_upstream": "https://github.com/veronica0206/gtheory4llm", "_commit": { "id": "e45cfa40dad8b99a17b77cea5397d61fb64f3c90", "author": "Jin (Veronica) Liu ", "committer": "Jin (Veronica) Liu ", "message": "Set 0.5.0 publication metadata\n", "time": 1791685599 }, "_maintainer": { "name": "Jin Liu", "email": "veronica.liu0206@gmail.com", "login": "veronica0206", "description": "👩‍🎓 Ph.D. in Biostatistics | Evaluating when you can trust LLM outputs | Scaling expert judgment into open-source AI skill systems | NLP | Longitudinal method", "uuid": 48883831 }, "_distro": "resolute", "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 4.5.0", "role": "Depends" }, { "package": "Matrix", "version": ">= 1.6.0", "role": "Imports" }, { "package": "OpenMx", "version": ">= 2.22.11", "role": "Imports" }, { "package": "graphics", "role": "Imports" }, { "package": "methods", "role": "Imports" }, { "package": "stats", "role": "Imports" }, { "package": "utils", "role": "Imports" }, { "package": "lme4", "role": "Suggests" }, { "package": "ordinal", "role": "Suggests" }, { "package": "knitr", "role": "Suggests" }, { "package": "rmarkdown", "role": "Suggests" } ], "_owner": "veronica0206", "_selfowned": true, "_usedby": 0, "_updates": [ { "week": "2026-37", "n": 16 }, { "week": "2026-38", "n": 49 }, { "week": "2026-39", "n": 13 }, { "week": "2026-40", "n": 1 }, { "week": "2026-41", "n": 40 } ], "_tags": [ { "name": "v0.0.6", "date": "2026-09-12" }, { "name": "v0.0.7", "date": "2026-09-13" }, { "name": "v0.1.0", "date": "2026-09-14" }, { "name": "v0.2.0", "date": "2026-09-23" }, { "name": "v0.4.0", "date": "2026-10-06" }, { "name": "v0.4.1", "date": "2026-10-07" }, { "name": "v0.5.0", "date": "2026-10-11" } ], "_stars": 0, "_contributors": [ { "user": "veronica0206", "count": 128, "uuid": 48883831 } ], "_userbio": { "uuid": 48883831, "type": "user", "name": "Jin (Veronica) Liu", "followers": 34, "description": "👩‍🎓 Ph.D. in Biostatistics | Evaluating when you can trust LLM outputs | Scaling expert judgment into open-source AI skill systems | NLP | Longitudinal method" }, "_downloads": { "count": 141, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/Gtheory4LLM" }, "_devurl": "https://github.com/veronica0206/gtheory4llm", "_searchresults": 0, "_rbuild": "4.6.1", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/Gtheory4LLM.html", "extra/manual.pdf", "extra/NEWS.html", "extra/NEWS.txt", "extra/readme.html", "extra/readme.md" ], "_homeurl": "https://github.com/veronica0206/gtheory4llm", "_realowner": "veronica0206", "_cranurl": true, "_releases": [ { "version": "0.2.0", "date": "2026-10-04" } ], "_exports": [ "gt_batch", "gt_batch_dstudy", "gt_batch_reliability", "gt_component_vcov", "gt_components", "gt_control", "gt_design", "gt_diagnostics", "gt_dstudy", "gt_dstudy_target", "gt_example", "gt_export_report", "gt_family", "gt_fit", "gt_pilot_plan", "gt_plan", "gt_preflight", "gt_reliability", "gt_report", "gt_score", "gt_simulate" ], "_help": [ { "page": "Gtheory4LLM-package", "title": "Generalizability Theory for LLM Subjective Tasks", "topics": [ "Gtheory4LLM-package", "Gtheory4LLM" ] }, { "page": "gt_batch", "title": "Declare That Items Were Annotated Several to a Call", "topics": [ "gt_batch" ] }, { "page": "gt_batch_dstudy", "title": "Compare Supplied Allocations for a Fixed Gaussian Batch Layout", "topics": [ "as.data.frame.gt_batch_dstudy", "gt_batch_dstudy", "print.gt_batch_dstudy" ] }, { "page": "gt_batch_reliability", "title": "Project Gaussian Score Error Conditional on a Fixed Batch Layout", "topics": [ "as.data.frame.gt_batch_reliability", "gt_batch_reliability", "print.gt_batch_reliability" ] }, { "page": "gt_components", "title": "Extract Source Covariances or Correlations", "topics": [ "gt_components" ] }, { "page": "gt_control", "title": "Configure Gaussian and Discrete Fitting and Acceptance Checks", "topics": [ "gt_control" ] }, { "page": "gt_design", "title": "Declare a Generalizability-Theory Random-Source Design", "topics": [ "gt_design" ] }, { "page": "gt_diagnostics", "title": "Inspect Numerical Acceptance and Backend Diagnostics", "topics": [ "gt_diagnostics", "print.gt_diagnostics", "print.gt_diagnostic_stage", "print.gt_diagnostic_stages" ] }, { "page": "gt_dstudy", "title": "Evaluate Balanced Measurement Allocations", "topics": [ "as.data.frame.gt_dstudy", "gt_dstudy", "gt_dstudy_target", "plot.gt_dstudy", "print.gt_dstudy" ] }, { "page": "gt_example", "title": "Load Modeling Tables from Three LLM-Annotated Datasets", "topics": [ "gt_example" ] }, { "page": "gt_family", "title": "Specify a Gaussian, Binary, Ordinal, or Unordered Categorical Outcome", "topics": [ "gt_family" ] }, { "page": "gt_fit", "title": "Fit Univariate or Joint Multivariate Generalizability Models", "topics": [ "gt_fit", "print.gt_fit", "summary.gt_fit" ] }, { "page": "gt_fit_methods", "title": "Standard Extractors for Fitted Generalizability Models", "topics": [ "coef.gt_fit", "gt_component_vcov", "gt_fit_methods", "logLik.gt_fit", "nobs.gt_fit", "vcov.gt_fit" ] }, { "page": "gt_pilot_plan", "title": "Evaluate Pilot Precision under a Gaussian Parameter Scenario", "topics": [ "gt_pilot_plan" ] }, { "page": "gt_plan", "title": "Rank a Bounded Set of D-Study Allocations by User-Supplied Cost", "topics": [ "as.data.frame.gt_plan", "gt_plan", "print.gt_plan" ] }, { "page": "gt_preflight", "title": "Inspect Data, Source Dimensions, and Backend Limits Before Fitting", "topics": [ "gt_preflight", "plot.gt_preflight", "print.gt_preflight" ] }, { "page": "gt_reliability", "title": "Compute Balanced G and Phi Coefficients", "topics": [ "as.data.frame.gt_reliability", "gt_reliability", "print.gt_reliability" ] }, { "page": "gt_report", "title": "Create and Export a Portable Aggregate Analysis Report", "topics": [ "gt_export_report", "gt_report", "print.gt_report" ] }, { "page": "gt_score", "title": "Declare Composite-Score Weights", "topics": [ "gt_score" ] }, { "page": "gt_simulate", "title": "Simulate a New Balanced Gaussian Annotation Panel", "topics": [ "gt_simulate" ] }, { "page": "gtheory_datasets", "title": "Three LLM Annotation Datasets: Design, Coding, and Sources", "topics": [ "Gtheory4LLM-datasets", "gtheory_datasets" ] }, { "page": "gtheory_drug_review", "title": "LLM Drug-Review Sentiment and Four Aspect Annotations", "topics": [ "drug_review", "drug_review_4aspect", "gtheory_drug_review" ] }, { "page": "gtheory_hate_speech", "title": "LLM Hate-Speech Annotations of HateXplain Texts", "topics": [ "gtheory_hate_speech", "hate_speech" ] }, { "page": "gtheory_mental_health", "title": "LLM Mental-Health Text Annotations in Five Outcome Sets", "topics": [ "gtheory_mental_health", "mental_health_3group", "mental_health_3L", "mental_health_6flag", "mental_health_7L", "mental_health_nominal" ] }, { "page": "plot.gt_reliability", "title": "Plot Reliability Coefficients and Available Intervals", "topics": [ "plot.gt_reliability" ] }, { "page": "print.summary.gt_fit", "title": "Print a Concise Generalizability Model Summary", "topics": [ "print.summary.gt_fit" ] } ], "_readme": "https://github.com/veronica0206/gtheory4llm/raw/HEAD/README.md", "_rundeps": [ "BH", "cli", "digest", "lattice", "lifecycle", "MASS", "Matrix", "mvtnorm", "OpenMx", "Rcpp", "RcppEigen", "RcppParallel", "rlang", "rpf", "StanHeaders" ], "_vignettes": [ { "source": "LLM-workflow.Rmd", "filename": "LLM-workflow.html", "title": "How many evaluators, prompts, and runs?", "engine": "knitr::rmarkdown", "headings": [ "1. Define the quantity and the sampling design", "2. Inspect feasibility before fitting", "3. Fit and inspect acceptance", "4. Compute relative and absolute reliability", "5. Compare measurement budgets", "6. Save a portable analysis report", "7. Fixed facets", "8. Match other outcomes to their observation model", "Where to read next" ], "created": "2026-09-13 03:32:52", "modified": "2026-10-04 21:42:20", "commits": 3 }, { "source": "study-planning.Rmd", "filename": "study-planning.html", "title": "Plan annotation effort, costs, and pilot precision", "engine": "knitr::rmarkdown", "headings": [ "Fit a small Gaussian pilot", "Compare costs within a declared allocation grid", "Make the batch target explicit", "Examine pilot precision for one final protocol" ], "created": "2026-10-09 04:54:37", "modified": "2026-10-09 19:17:40", "commits": 3 } ], "_score": 4.698970004336019, "_indexed": true, "_nocasepkg": "gtheory4llm", "_universes": [ "veronica0206" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "0.5.0", "date": "2026-10-11T09:10:49.000Z", "distro": "resolute", "commit": "e45cfa40dad8b99a17b77cea5397d61fb64f3c90", "fileid": "https://r2.ropensci.org/d2c194987a28f0f33fe59ab4b96d95a935c90d1bbbd6b62349a81b1b006c32f9", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/veronica0206/actions/runs/38127088638" }, { "r": "4.6.1", "os": "linux", "version": "0.5.0", "date": "2026-10-11T09:10:58.000Z", "distro": "resolute", "commit": "e45cfa40dad8b99a17b77cea5397d61fb64f3c90", "fileid": "https://r2.ropensci.org/a60768d9369e99e1acfdb8671a9e3a777164d9ca8cceb360094bb3b4909724ef", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/veronica0206/actions/runs/38127088638" }, { "r": "4.5.3", "os": "mac", "version": "0.5.0", "date": "2026-10-11T09:14:26.000Z", "commit": "e45cfa40dad8b99a17b77cea5397d61fb64f3c90", "fileid": "https://r2.ropensci.org/567c498aa39744da98fcf8925140709f5ee159e99917527c880f6626becf15d0", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/veronica0206/actions/runs/38127088638" }, { "r": "4.6.1", "os": "mac", "version": "0.5.0", "date": "2026-10-11T09:11:52.000Z", "commit": "e45cfa40dad8b99a17b77cea5397d61fb64f3c90", "fileid": "https://r2.ropensci.org/5e7b6cbda2cd6cd5331a84f3da37b593de49ee8f0ef5e6bb02bf185467f54066", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/veronica0206/actions/runs/38127088638" }, { "r": "4.6.0", "os": "wasm", "version": "0.5.0", "date": "2026-10-11T09:11:02.000Z", "commit": "e45cfa40dad8b99a17b77cea5397d61fb64f3c90", "fileid": "https://r2.ropensci.org/a7cd27d0f166adcacded89de60bbcd1a38bc9defaca6ca69d33e2af0420e1b99", "status": "success", "buildurl": "https://github.com/r-universe/veronica0206/actions/runs/38127088638" }, { "r": "4.7.0", "os": "win", "version": "0.5.0", "date": "2026-10-11T09:10:25.000Z", "commit": "e45cfa40dad8b99a17b77cea5397d61fb64f3c90", "fileid": "https://r2.ropensci.org/149d0e2409d7e83ab9cadfc5474cf531fd65ef60c222986625bd9e7fdb1165ad", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/veronica0206/actions/runs/38127088638" }, { "r": "4.5.3", "os": "win", "version": "0.5.0", "date": "2026-10-11T09:10:28.000Z", "commit": "e45cfa40dad8b99a17b77cea5397d61fb64f3c90", "fileid": "https://r2.ropensci.org/a03e0e9827b031287fb95a429e1f2accc9c9db54484a52b05f9d9882cbe6a9e1", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/veronica0206/actions/runs/38127088638" }, { "r": "4.6.1", "os": "win", "version": "0.5.0", "date": "2026-10-11T09:10:04.000Z", "commit": "e45cfa40dad8b99a17b77cea5397d61fb64f3c90", "fileid": "https://r2.ropensci.org/15208475f4522f788e8e5d3af297dc8b9e36228c7be8d9693f1ef994696a31e9", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/veronica0206/actions/runs/38127088638" } ] }