diff --git a/Project.toml b/Project.toml index df5459c..02d7a7d 100644 --- a/Project.toml +++ b/Project.toml @@ -16,15 +16,12 @@ Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" EffectSizes = "e248de7e-9197-5860-972e-353a2af44d75" ElasticArrays = "fdbdab4c-e67f-52f5-8c3f-e7b388dad3d4" -FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b" FittedItemBanks = "3f797b09-34e4-41d7-acf6-3302ae3248a5" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" HypothesisTests = "09f84164-cd44-5f33-b23f-e6b0d136a0d5" -Interpolations = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" Lazy = "50d2b5c4-7a5e-59d5-8109-a42b560f39c0" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" LogarithmicNumbers = "aa2f6b4e-9042-5d33-9679-40d3a6b85899" -MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" Mmap = "a63ad114-7e13-5084-954f-fe012c677804" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" PrettyPrinting = "54e16d92-306c-5ea0-a30b-337be88ac337" @@ -33,8 +30,6 @@ PsychometricsBazaarBase = "b0d9cada-d963-45e9-a4c6-4746243987f1" QuickHeaps = "30b38841-0f52-47f8-a5f8-18d5d4064379" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Reexport = "189a3867-3050-52da-a836-e630ba90ab69" -Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46" -SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" StatsFuns = "4c63d2b9-4356-54db-8cca-17b64c39e42c" @@ -55,15 +50,12 @@ Distributions = "^0.25.88" DocStringExtensions = " ^0.9" EffectSizes = "^1.0.1" ElasticArrays = "1.2.12" -FillArrays = "0.13, 1.5.0" FittedItemBanks = "^0.7.3" ForwardDiff = "1" HypothesisTests = "^0.10.12, ^0.11.0" -Interpolations = "^0.14, ^0.15" Lazy = "0.15" LinearAlgebra = "^1.11" LogarithmicNumbers = "1" -MacroTools = "^0.5.6" Mmap = "^1.11" PrecompileTools = "1.2.1" PrettyPrinting = "0.4.2" @@ -72,8 +64,6 @@ PsychometricsBazaarBase = "^0.8.7" QuickHeaps = "0.2.2" Random = "^1.11" Reexport = "1" -Setfield = "^1" -SparseArrays = "^1.11" StaticArrays = "1" StatsBase = "^0.34" StatsFuns = "^0.9.15, ^1" diff --git a/profile/Project.toml b/profile/Project.toml deleted file mode 100644 index c984419..0000000 --- a/profile/Project.toml +++ /dev/null @@ -1,6 +0,0 @@ -[deps] -ComputerAdaptiveTesting = "5a0d4f34-1f62-4a66-80fe-87aba0485488" -PProf = "e4faabce-9ead-11e9-39d9-4379958e3056" -StatProfilerHTML = "a8a75453-ed82-57c9-9e16-4cd1196ecbf5" -BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" -Coverage = "a2441757-f6aa-5fb2-8edb-039e3f45d037" diff --git a/profile/clear_mems.sh b/profile/clear_mems.sh deleted file mode 100755 index 08986b2..0000000 --- a/profile/clear_mems.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -PATTERN="*.$1.mem" -find -name $PATTERN -exec rm -i {} \; \ No newline at end of file diff --git a/profile/does_quadgk_alloc.jl b/profile/does_quadgk_alloc.jl deleted file mode 100644 index ea74839..0000000 --- a/profile/does_quadgk_alloc.jl +++ /dev/null @@ -1,10 +0,0 @@ -using QuadGK -using Distributions -using Profile - -const int_tol = 1e-6 -const std_norm = Normal() -norm_cdf = x -> cdf(std_norm, x) -quadgk(norm_cdf, -10.0, 1.0, int_tol)[1] -Profile.clear_malloc_data() -quadgk(norm_cdf, -10.0, 1.0, int_tol)[1] diff --git a/profile/mv_mems.sh b/profile/mv_mems.sh deleted file mode 100755 index ee18c5a..0000000 --- a/profile/mv_mems.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -mkdir -p $2 -PATTERN="*.$1.mem" -find -name $PATTERN -exec mv {} $2/ \; \ No newline at end of file diff --git a/profile/next_items.jl b/profile/next_items.jl deleted file mode 100644 index ef8b5fc..0000000 --- a/profile/next_items.jl +++ /dev/null @@ -1,204 +0,0 @@ -using ComputerAdaptiveTesting.DummyData: dummy_3pl, - dummy_mirt_4pl, std_normal, std_mv_normal -using ComputerAdaptiveTesting.Responses -using ComputerAdaptiveTesting.Aggregators -using ComputerAdaptiveTesting.Integrators -using ComputerAdaptiveTesting.NextItemRules -import Profile -import PProf -using Distributions: Normal -using ArgParse -using StatProfilerHTML -using StatsBase: sample -using BenchmarkTools - -function get_ability_estimator(multidim) - if multidim - integrator = MultiDimFixedGKIntegrator([-6.0, -6.0, -6.0], [6.0, 6.0, 6.0]) - dist = std_mv_normal(3) - else - integrator = FixedGKIntegrator(-6.0, 6.0) - dist = Normal() - end - return PosteriorAbilityEstimator(dist, integrator) -end - -function prepare_empty(item_bank, actual_responses, ability_tracker) - responses = TrackedResponses(BareResponses([], []), - item_bank, - ability_tracker) - (item_bank, actual_responses, responses) -end - -function prepare_0(ability_estimator) - (item_bank, question_labels_, abilities_, actual_responses) = dummy_3pl(; - num_questions = 100, - num_testees = 1) - prepare_empty(item_bank, actual_responses, PointAbilityTracker(ability_estimator, NaN)) -end - -function prepare_0_mirt(ability_estimator) - (item_bank, question_labels_, abilities_, actual_responses) = dummy_mirt_4pl(3; - num_questions = 100, - num_testees = 1) - prepare_empty(item_bank, - actual_responses, - PointAbilityTracker(ability_estimator, [NaN, NaN, NaN])) -end - -function prepare_50(ability_estimator) - (item_bank, question_labels_, abilities_, actual_responses) = dummy_3pl(; - num_questions = 100, - num_testees = 1) - idxs = sample(1:100, 50) - responses = TrackedResponses(BareResponses(idxs, - actual_responses[idxs, 1]), - item_bank, - PointAbilityTracker(ability_estimator, NaN)) - (item_bank, actual_responses, responses) -end - -function run_all(dummy_data, objective) - (item_bank, actual_responses, responses) = dummy_data - track!(responses) - criterion_state = init_thread(objective, responses) - for item_idx in eachindex(item_bank) - objective(criterion_state, responses, item_idx) - end -end - -function run_single(dummy_data, objective) - (item_bank, actual_responses, responses) = dummy_data - criterion_state = init_thread(objective, responses) - objective(criterion_state, responses, 1) -end - -function profile_objective(run_profile::RunProfile, - pre_bench::PrepBench, - run_bench::RunBench, - objective::Objective, - ability_estimator) where {RunProfile, PrepBench, RunBench, Objective} - dummy_data = pre_bench(ability_estimator) - run() = run_bench(dummy_data, objective) - @info "init run" - run() - @info "benchmark run" - run_profile(run) -end - -#= -if isdefined(Profile, :Allocs) - run_pprof_allocs = function(run) - Profile.Allocs.@profile run() - prof = Profile.Allocs.fetch() - PProf.Allocs.pprof(prof) - end -end -=# - -function get_cmdline() - if Sys.iswindows() - String.(split(unsafe_string(ccall(:GetCommandLineA, Cstring, ())), " ")) - elseif Sys.isapple() - String.(split(strip(read(`/bin/ps -p $(getpid()) -o command=`, String)), " ")) - elseif Sys.isunix() - String.(split(read(joinpath("/", "proc", string(getpid()), "cmdline"), String), - "\x00"; - keepempty = false)) - else - j_cmd = String.(split(Base.julia_cmd(), " ")) - args_joined = join(ARGS, " ") - [j_cmd..., PROGRAM_FILE, args_joined...] - end -end - -function run_track_allocs(run_bench::F) where {F} - Profile.clear_malloc_data() - run_bench() -end - -function run_profile(run::F) where {F} - Profile.@profile run() - prof = Profile.fetch() - PProf.pprof(prof) -end - -function run_time(run::F) where {F} - @time run() -end - -function run_btime(run_bench::F) where {F} - t = @benchmark $(run_bench)() evals=1000 samples=1000 - dump(t) -end - -function run_statprofilerhtml(run::F) where {F} - @profilehtml run() -end - -PROFILERS = Dict("track_allocs" => run_track_allocs, - "profile" => run_profile, - "time" => run_time, - "btime" => run_btime, - "profilehtml" => run_statprofilerhtml) - -BENCHES = Dict("all0" => (prepare_0, run_all), - "allmirt" => (prepare_0_mirt, run_all), - "all50" => (prepare_50, run_all), - "single50" => (prepare_50, run_single)) -#= -if isdefined(Profile, :Allocs) - PROFILERS["pprof_allocs"] = run_pprof_allocs -end -=# - -function objective(next_item_rule::NextItemRule) - next_item_rule.criterion -end - -function get_next_item_rule(rule_name, ability_estimator) - if rule_name == "drule" - NextItemRule(DRuleItemCriterion(ability_estimator)) - else - catr_next_item_aliases[rule_name](ability_estimator) - end -end - -function main() - settings = ArgParseSettings() - @add_arg_table settings begin - "profiling_mode" - help = "The profiling mode to use. Can be 'pprof_allocs', 'track_allocs', 'profile', or 'time'." - required = true - "next_item_rule" - help = "The next item rule to use" - required = true - "bench" - help = "The benchmark to use" - required = true - end - args = parse_args(settings) - ability_estimator = get_ability_estimator(args["next_item_rule"] == "drule") - point_ability_estimator = ModeAbilityEstimator(ability_estimator) - next_item_rule = get_next_item_rule(args["next_item_rule"], point_ability_estimator) - if args["profiling_mode"] == "track_allocs" && !haskey(ENV, "TRACK_ALLOCS") - cmdline = get_cmdline() - insert!(cmdline, - findfirst(x -> endswith(x, ".jl"), cmdline), - "--track-allocation=all") - cmd = Cmd(Cmd(cmdline), env = ("TRACK_ALLOCS" => "TRUE",)) - @info "Running subprocess to track allocs" cmd - Base.run(cmd) - return - end - (pre_bench, run_bench) = BENCHES[args["bench"]] - profile_objective(PROFILERS[args["profiling_mode"]], - pre_bench, - run_bench, - objective(next_item_rule), - point_ability_estimator) -end - -if abspath(PROGRAM_FILE) == @__FILE__ - main() -end diff --git a/profile/print_mems.jl b/profile/print_mems.jl deleted file mode 100644 index 22cb033..0000000 --- a/profile/print_mems.jl +++ /dev/null @@ -1,3 +0,0 @@ -using Coverage - -display(analyze_malloc([ARGS[1]])) diff --git a/src/Aggregators/slow.jl b/src/Aggregators/slow.jl deleted file mode 100644 index bda960d..0000000 --- a/src/Aggregators/slow.jl +++ /dev/null @@ -1,19 +0,0 @@ -function slow_int_abil_posterior_given_resps{F}(f::F, - responses::AbstractVector{Response}, - items::AbstractItemBank; - lo = 0.0, - hi = 10.0) where {F} - quadgk((θ -> f(θ) * abil_posterior_given_resps(responses, items, θ)), lo, hi, int_tol)[1] -end - -function slow_max_abil_posterior_given_resps(f::Function, - responses::AbstractVector{Response}, - items::AbstractItemBank; - lo = 0.0, - hi = 10.0) - optimize((θ_arr -> -abil_posterior_given_resps(responses, items, first(θ_arr))), - lo, - hi, - NelderMead(), - Optim.Options(g_tol = optim_tol))[1] -end diff --git a/src/Compat/CatR.jl b/src/Compat/CatR.jl index c17cb6e..8c6cc56 100644 --- a/src/Compat/CatR.jl +++ b/src/Compat/CatR.jl @@ -61,23 +61,6 @@ end const ability_estimator_aliases = _ability_estimator_aliases() -#= - for (resp_exp, resp_exp_nick) in resp_exp_nick_pairs - next_item_rule = NextItemRule( - ExpectationBasedItemCriterion(resp_exp, AbilityVariance(numtools.integrator, distribution_estimator(abil_est))) - ) - next_item_rule = preallocate(next_item_rule) - est_next_item_rule_pairs[Symbol("$(abil_est_str)_mepv_$(resp_exp_nick)")] = (abil_est, next_item_rule) - next_item_rule = NextItemRule( - ExpectationBasedItemCriterion(resp_exp, InformationItemCriterion(abil_est)) - ) - next_item_rule = preallocate(next_item_rule) - est_next_item_rule_pairs[Symbol("$(abil_est_str)_mei_$(resp_exp_nick)")] = (abil_est, next_item_rule) - end - est_next_item_rule_pairs[Symbol("$(abil_est_str)_mi")] = (abil_est, InformationItemCriterion(abil_est)) -=# - - function setup_integrator(lo=-4.0, hi=4.0, pts=33) Integrators.MidpointIntegrator(range(lo, hi, pts)) end diff --git a/src/NextItemRules/NextItemRules.jl b/src/NextItemRules/NextItemRules.jl index 0016f19..edec020 100644 --- a/src/NextItemRules/NextItemRules.jl +++ b/src/NextItemRules/NextItemRules.jl @@ -64,6 +64,12 @@ export ObservedInformationPointwiseItemCriterion export RawEmpiricalInformationPointwiseItemCriterion export EmpiricalInformationPointwiseItemCriterion +public PointwiseNextItemRule, PointwiseFirstNextItemRule +public WeightedStateMultiCriterion, WeightedItemMultiCriterion +public GreedyForcedContentBalancer +public PosteriorExpectedKLInformationItemCriterion +public alt_expected_1d_item_information, alt_expected_mirt_item_information, alt_expected_mirt_3pl_item_information + # Prelude include("./prelude/abstract.jl") include("./prelude/next_item_rule.jl") diff --git a/src/NextItemRules/criteria/pointwise/information_support.jl b/src/NextItemRules/criteria/pointwise/information_support.jl index 5f4dd30..6442e32 100644 --- a/src/NextItemRules/criteria/pointwise/information_support.jl +++ b/src/NextItemRules/criteria/pointwise/information_support.jl @@ -30,42 +30,6 @@ function log_resp(ir::ItemResponse{<:CdfMirtItemBank}, val, θ) end end -#= -# XXX: Not sure if this is optimal numerically or speed wise -- possibly it -# would be better to just transform to linear space in this case? -@inline function log_transform_irf_y(guess, slip, y) - # log space version of guess + irf_size(guess, slip) * y - logaddexp(log(guess), log(irf_size(guess, slip)) + y) -end - -@inline function log_transform_irf_y(ir::ItemResponse{<:GuessItemBank}, response, y) - guess = y_offset(ir.item_bank, ir.index) - if response - log_transform_irf_y(guess, 0.0, y) - else - log_transform_irf_y(0.0, guess, y) - end -end - -@inline function log_transform_irf_y(ir::ItemResponse{<:SlipItemBank}, response, y) - slip = y_offset(ir.item_bank, ir.index) - if response - log_transform_irf_y(0.0, slip, y) - else - log_transform_irf_y(slip, 0.0, y) - end -end - -function log_resp_vec(ir::ItemResponse{<:AnySlipOrGuessItemBank}, θ) - r = log_resp_vec(inner_item_response(ir), θ) - SVector(log_transform_irf_y(ir, false, r[1]), log_transform_irf_y(ir, true, r[2])) -end - -function log_resp(ir::ItemResponse{<:AnySlipOrGuessItemBank}, val, θ) - log_transform_irf_y(ir, val, log_resp(inner_item_response(ir), val, θ)) -end -=# - log_resp(ir::ItemResponse{<:GuessAndSlipItemBank}, response, θ) = log(resp(ir, response, θ)) log_resp(ir::ItemResponse{<:GuessAndSlipItemBank}, θ) = log(resp(ir, θ)) log_resp_vec(ir::ItemResponse{<:GuessAndSlipItemBank}, θ) = log.(resp_vec(ir, θ)) diff --git a/src/NextItemRules/criteria/pointwise/kl.jl b/src/NextItemRules/criteria/pointwise/kl.jl index efaf115..a19e7eb 100644 --- a/src/NextItemRules/criteria/pointwise/kl.jl +++ b/src/NextItemRules/criteria/pointwise/kl.jl @@ -12,6 +12,9 @@ struct PosteriorExpectedKLInformationItemCriterion{ DistributionEstimatorT <: DistributionAbilityEstimator, IntegratorT <: AbilityIntegrator } <: PointwiseItemCriterion + point_estimator::PointEstimatorT + distribution_estimator::DistributionEstimatorT + integrator::IntegratorT end function PosteriorExpectedKLInformationItemCriterion(bits...) diff --git a/src/NextItemRules/porcelain/aliases.jl b/src/NextItemRules/porcelain/aliases.jl deleted file mode 100644 index e69de29..0000000 diff --git a/src/NextItemRules/strategies/random.jl b/src/NextItemRules/strategies/random.jl index 4f5965d..8d2ad2d 100644 --- a/src/NextItemRules/strategies/random.jl +++ b/src/NextItemRules/strategies/random.jl @@ -10,18 +10,6 @@ administered. rng::RandomT = Xoshiro() end -#= -function get_rng(bits...) - @returnsome find1_instance(AbstractRNG, bits) - @returnsome find1_type(AbstractRNG, bits) typ -> typ() - Xoshiro() -end - -function RandomNextItemRule(bits...) - RandomNextItemRule(rng=get_rng(bits...)) -end -=# - function best_item(rule::RandomNextItemRule, responses::TrackedResponses, items) # TODO: This is not efficient item_idxes = Set(1:length(items)) diff --git a/src/Stateful.jl b/src/Stateful.jl index f8482e6..c5621cd 100644 --- a/src/Stateful.jl +++ b/src/Stateful.jl @@ -155,32 +155,6 @@ model backing the CAT. """ function item_response_functions end -## Running the CAT -function Sim.run_cat(cat_config::CatLoop{RulesT}, - ib_labels = nothing) where {RulesT <: StatefulCat} - (; stateful_cat, get_response, new_response_callback) = cat_config - while true - next_index = next_item(stateful_cat) - next_label = item_label(ib_labels, next_index) - @debug "Querying" next_index next_label - response = get_response(next_index, next_label) - @debug "Got response" response - add_response!(stateful_cat, next_index, response) - terminating = termination_condition(responses, item_bank) - if new_response_callback !== nothing - new_response_callback(get_responses(responses), terminating) - end - if terminating - @debug "Met termination condition" - break - end - end - return ( - get_responses(stateful_cat), - get_ability(stateful_cat) - ) -end - ## TODO: Materialise the cat into a decsision tree """