#include "../../include/rcore.h"
#include "../../core/allocations.h"
#include "../../core/correlation.h"
#include "../../core/covariance.matrix.h"
#include "../../core/data.table.h"
#include "../../core/math.functions.h"
#include "../../core/sets.h"
#include "../../include/globals.h"
#include "../patterns.h"
#include "../tests.h"

void update_pvalue_range(double pvalue, double *min, double *max) {

  *min = pvalue < *min ? pvalue : *min;
  *max = pvalue > *max ? pvalue : *max;

}/*UPDATE_PVALUE_RANGE*/

ast_result ast_result_make(double pvalue, double min_pvalue, double max_pvalue,
    bool separated, const char **nodes, int nnodes) {

ast_result res;

  /* fill the p-value fields; when the variables were separated, take an owned
   * copy of the (borrowed) separating-set name pointers. */
  res.pvalue = pvalue;
  res.min_pvalue = min_pvalue;
  res.max_pvalue = max_pvalue;
  res.separated = separated;
  res.ndsep = separated ? nnodes : 0;
  res.dsep = NULL;

  if (separated && (nnodes > 0)) {

    res.dsep = Calloc1D(nnodes, sizeof(char *));
    for (int i = 0; i < nnodes; i++)
      res.dsep[i] = nodes[i];

  }/*THEN*/

  return res;

}/*AST_RESULT_MAKE*/

/* parametric tests for discrete variables. */
ast_result ast_discrete(tabular dtx, tabular dty, tabular dtz, int nf,
    int minsize, int maxsize, test_e test, double a, bool debugging) {

int *xptr = dtx.dcol[0], *yptr = dty.dcol[0], *zptr = NULL, *subset = NULL;
int i = 0, cursize = 0, llx = dtx.nlvl[0], lly = dty.nlvl[0], llz = 0;
double statistic = 0, pvalue = 0, df = 0, min_pvalue = 1, max_pvalue = 0;
bool xy_latent = tabular_has_latent(dtx) || tabular_has_latent(dty);
ast_result retval;
tabular sub = { 0 };

  /* allocate a second data table to hold the conditioning variables. */
  sub = empty_tabular(dtz.m.nobs, dtz.m.ncols, 0);
  /* allocate the parents' configurations. */
  zptr = Calloc1D(dtz.m.nobs, sizeof(int));

  for (cursize = imax(1, minsize); cursize <= maxsize; cursize++) {

    /* allocate and initialize the subset. */
    subset = Calloc1D(cursize + nf, sizeof(int));
    /* initialize the first subset (assumes fixed variables are the first nf). */
    first_subset(subset + nf, cursize, nf);
    for (i = 0; i < nf; i++)
      subset[i] = i;

    /* iterate over subsets. */
    do {

      /* prepare the current subset. */
      tabular_subset_columns(&dtz, &sub, subset, cursize + nf);

      if (xy_latent || tabular_has_latent(sub)) {

        /* a latent variable makes the contingency table empty: the test is
         * degenerate and returns independence without touching the data. */
        pvalue = 1;
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/
      else {

      /* construct the parents' configurations. */
      c_fast_config(sub.dcol, sub.m.nobs, cursize + nf, sub.nlvl, zptr, &llz, 1);

      if (test == MI || test == MI_ADF || test == X2 || test == X2_ADF) {

        /* mutual information and Pearson's X^2 asymptotic tests. */
        statistic = c_cchisqtest(xptr, llx, yptr, lly, zptr, llz, sub.m.nobs, &df,
                      test, (test == MI) || (test == MI_ADF));
        pvalue = pchisq(statistic, df, FALSE, FALSE);
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/
      else if (test == MI_SH) {

        /* shrinkage mutual information test. */
        statistic = c_shcmi(xptr, llx, yptr, lly, zptr, llz, sub.m.nobs, &df, TRUE);
        pvalue = pchisq(statistic, df, FALSE, FALSE);
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/
      else if (test == JT) {

        /* Jonckheere-Terpstra test. */
        statistic = c_cjt(xptr, llx, yptr, lly, zptr, llz, sub.m.nobs);
        pvalue = 2 * pnorm(fabs(statistic), 0, 1, FALSE, FALSE);
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/

      }/*ELSE*/

      /* increment the test counter. */
      test_counter++;

      if (debugging)
        debug_test_outcome(dtx.m.names[0], dty.m.names[0], sub.m.names,
          sub.m.ncols, pvalue, a);

      if (pvalue > a) {

        retval = ast_result_make(pvalue, min_pvalue, max_pvalue, TRUE,
                   sub.m.names, sub.m.ncols);

        Free1D(subset);
        Free1D(zptr);
        FreeTAB(sub);

        return retval;

      }/*THEN*/

    } while (next_subset(subset + nf, cursize, dtz.m.ncols - nf, nf));

    Free1D(subset);

  }/*FOR*/

  Free1D(zptr);
  FreeTAB(sub);

  return ast_result_make(pvalue, min_pvalue, max_pvalue, FALSE, NULL, 0);

}/*AST_DISCRETE*/

/* parametric tests for Gaussian variables (for complete data). */
static ast_result ast_gaustests_complete(tabular dt, int nf, int minsize,
    int maxsize, double a, bool debugging, test_e test) {

int i = 0, cursize = 0, *subset = NULL, *prev = NULL;
double statistic = 0, lambda = 0, df = 0;
double pvalue = 0, min_pvalue = 1, max_pvalue = 0;
ast_result retval;
tabular sub = { 0 };
covariance cov = { 0 }, master = { 0 };

  /* allocate a second data table to hold the conditioning variables. */
  sub = empty_tabular(dt.m.nobs, 0, dt.m.ncols);
  sub.mean = Calloc1D(dt.m.ncols, sizeof(double));

  for (cursize = imax(1, minsize); cursize <= maxsize; cursize++) {

    /* compute the degrees of freedom for correlation and mutual information. */
    df = gaussian_cdf(test, dt.m.nobs, cursize + nf);

    if (df < 1) {

      /* if there are not enough degrees of freedom, return independence. */
      warning("trying to do a conditional independence test with zero degrees of freedom.");

      pvalue = min_pvalue = max_pvalue = 1;

      if (debugging) {

        Rprintf("    > node %s is independent from %s given any conditioning set of size %d ",
          dt.m.names[0], dt.m.names[1], cursize + nf);
        Rprintf("(p-value: %g).\n", pvalue);

      }/*THEN*/

      /* the conditioning set is the first possible conditioning set, which
       * comprises the first columns of the data table. */
      retval = ast_result_make(pvalue, min_pvalue, max_pvalue, TRUE,
                 dt.m.names + 2, cursize + nf);

      FreeTAB(sub);

      return retval;

    }/*THEN*/

    /* allocate the subset index array and a copy of the previous one, used to
     * spot which conditioning columns next_subset() changes between subsets. */
    subset = Calloc1D(cursize + nf + 2, sizeof(int));
    prev = Calloc1D(cursize + nf + 2, sizeof(int));
    /* the master covariance is maintained incrementally and never decomposed;
     * cov holds a fresh copy of it for each pcor, whose SVD overwrites it. */
    master = new_covariance(cursize + nf + 2, FALSE);
    cov = new_covariance(cursize + nf + 2, TRUE);
    /* initialize the first subset. */
    first_subset(subset + nf + 2, cursize, nf + 2);
    for (i = 0; i < nf + 2; i++)
      subset[i] = i;

    /* compute the first subset's covariance in full; later subsets only refresh
     * the conditioning columns that actually changed. */
    tabular_subset_columns(&dt, &sub, subset, cursize + nf + 2);
    c_covmat(sub.ccol, sub.mean, sub.m.nobs, sub.m.ncols, master, 0);
    memcpy(prev, subset, (cursize + nf + 2) * sizeof(int));

    do {

      /* prepare the current subset and refresh the covariance rows/columns that
       * next_subset() changed (none on the first pass); x, y and the fixed
       * conditioning columns never move, so only the selected block can change.
       * c_update_covmat() is an exact recompute, so the maintained matrix stays
       * bit-identical to a full c_covmat(). */
      tabular_subset_columns(&dt, &sub, subset, cursize + nf + 2);
      for (i = nf + 2; i < cursize + nf + 2; i++)
        if (subset[i] != prev[i])
          c_update_covmat(sub.ccol, sub.mean, i, sub.m.nobs, sub.m.ncols,
            master.mat);
      /* work on a copy: c_fast_pcor()'s SVD overwrites the matrix in place. */
      copy_covariance(&master, &cov);

      if (test == COR) {

        statistic = c_fast_pcor(cov, 0, 1, NULL, TRUE);
        statistic = cor_t_trans(statistic, df);
        pvalue = 2 * pt(fabs(statistic), df, FALSE, FALSE);
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/
      else if (test == MI_G) {

        statistic = c_fast_pcor(cov, 0, 1, NULL, TRUE);
        statistic = 2 * sub.m.nobs * cor_mi_trans(statistic);
        pvalue = pchisq(statistic, df, FALSE, FALSE);
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/
      else if (test == MI_G_SH) {

        lambda = covmat_lambda(sub.ccol, sub.mean, cov, sub.m.nobs, NULL,
                   sub.m.nobs);
        covmat_shrink(cov, lambda);
        statistic = c_fast_pcor(cov, 0, 1, NULL, TRUE);
        statistic = 2 * sub.m.nobs * cor_mi_trans(statistic);
        pvalue = pchisq(statistic, df, FALSE, FALSE);
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/
      else if (test == ZF) {

        statistic = c_fast_pcor(cov, 0, 1, NULL, TRUE);
        statistic = cor_zf_trans(statistic, df);
        pvalue = 2 * pnorm(fabs(statistic), 0, 1, FALSE, FALSE);
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/

      /* increment the test counter. */
      test_counter++;

      if (debugging)
        debug_test_outcome(sub.m.names[0], sub.m.names[1], sub.m.names + 2,
          sub.m.ncols - 2, pvalue, a);

      if (pvalue > a) {

        retval = ast_result_make(pvalue, min_pvalue, max_pvalue, TRUE,
                   sub.m.names + 2, sub.m.ncols - 2);

        Free1D(subset);
        Free1D(prev);
        FreeCOV(cov);
        FreeCOV(master);
        FreeTAB(sub);

        return retval;

      }/*THEN*/

      /* remember this subset to detect the next one's changed columns. */
      memcpy(prev, subset, (cursize + nf + 2) * sizeof(int));

    } while (next_subset(subset + nf + 2, cursize, dt.m.ncols - nf - 2, nf + 2));

    FreeCOV(cov);
    FreeCOV(master);
    Free1D(subset);
    Free1D(prev);

  }/*FOR*/

  FreeTAB(sub);

  return ast_result_make(pvalue, min_pvalue, max_pvalue, FALSE, NULL, 0);

}/*AST_GAUSTESTS_COMPLETE*/

/* parametric tests for Gaussian variables (for incomplete data). */
static ast_result ast_gaustests_with_missing(tabular dt, int nf, int minsize,
    int maxsize, double a, bool debugging, test_e test) {

int i = 0, cursize = 0, *subset = NULL, ncomplete = 0;
double statistic = 0, lambda = 0, df = 0;
double pvalue = 0, min_pvalue = 1, max_pvalue = 0;
double *mean = NULL;
bool *missing_xy = NULL, *missing_all = NULL;
ast_result retval;
tabular sub = { 0 };
covariance cov = { 0 };

  /* allocate a second data table to hold the conditioning variables. */
  sub = empty_tabular(dt.m.nobs, 0, dt.m.ncols);
  sub.mean = Calloc1D(dt.m.ncols, sizeof(double));
  /* allocate the missingness indicators. */
  missing_xy = Calloc1D(dt.m.nobs, sizeof(bool));
  tabular_incomplete_cases_range(&dt, missing_xy, 0, -1, 0, 1);

  for (cursize = imax(1, minsize); cursize <= maxsize; cursize++) {

    /* allocate a vector to store column means (from complete observations). */
    mean = Calloc1D(cursize + nf + 2, sizeof(double));
    /* allocate missingness indicators. */
    missing_all = Calloc1D(sub.m.nobs, sizeof(bool));
    /* allocate and initialize the subset indexes array. */
    subset = Calloc1D(cursize + nf + 2, sizeof(int));
    /* allocate the covariance matrix and the U, D, V matrix. */
    cov = new_covariance(cursize + nf + 2, TRUE);
    /* initialize the first subset. */
    first_subset(subset + nf + 2, cursize, nf + 2);
    for (i = 0; i < nf + 2; i++)
      subset[i] = i;

    do {

      /* prepare the current subset. */
      tabular_subset_columns(&dt, &sub, subset, cursize + nf + 2);
      /* a latent (fully unobserved) column leaves no complete cases, so skip the
       * covariance and fall straight through to the zero-complete-cases handling
       * below with the same result. */
      if (tabular_has_latent(sub))
        ncomplete = 0;
      else
        c_covmat_with_missing(sub.ccol, sub.m.nobs, sub.m.ncols, missing_xy,
          missing_all, mean, cov.mat, &ncomplete);

      /* compute the degrees of freedom for correlation and mutual information. */
      df = gaussian_cdf(test, ncomplete, cursize + nf);

      if ((ncomplete == 0) || (df < 1)) {

        /* if there are not enough degrees of freedom, return independence. */
        warning("trying to do a conditional independence test with zero degrees of freedom.");

        pvalue = min_pvalue = max_pvalue = 1;

        if (debugging) {

          Rprintf("    > node %s is independent from %s given any conditioning set of size %d ",
            dt.m.names[0], dt.m.names[1], cursize + nf);
          Rprintf("(p-value: %g).\n", pvalue);

        }/*THEN*/

        /* the conditioning set is the first possible conditioning set, which
         * comprises the first columns of the data table. */
        retval = ast_result_make(pvalue, min_pvalue, max_pvalue, TRUE,
                   dt.m.names + 2, cursize + nf);

        FreeTAB(sub);
        Free1D(missing_xy);
        Free1D(missing_all);
        Free1D(subset);
        Free1D(mean);
        FreeCOV(cov);

        return retval;

      }/*THEN*/

      if (test == COR) {

        statistic = c_fast_pcor(cov, 0, 1, NULL, TRUE);
        statistic = cor_t_trans(statistic, df);
        pvalue = 2 * pt(fabs(statistic), df, FALSE, FALSE);
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/
      else if (test == MI_G) {

        statistic = c_fast_pcor(cov, 0, 1, NULL, TRUE);
        statistic = 2 * ncomplete * cor_mi_trans(statistic);
        pvalue = pchisq(statistic, df, FALSE, FALSE);
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/
      else if (test == MI_G_SH) {

        lambda = covmat_lambda(sub.ccol, mean, cov, sub.m.nobs, missing_all,
                   ncomplete);
        covmat_shrink(cov, lambda);
        statistic = c_fast_pcor(cov, 0, 1, NULL, TRUE);
        statistic = 2 * ncomplete * cor_mi_trans(statistic);
        pvalue = pchisq(statistic, df, FALSE, FALSE);
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/
      else if (test == ZF) {

        statistic = c_fast_pcor(cov, 0, 1, NULL, TRUE);
        statistic = cor_zf_trans(statistic, df);
        pvalue = 2 * pnorm(fabs(statistic), 0, 1, FALSE, FALSE);
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/

      /* increment the test counter. */
      test_counter++;

      if (debugging)
        debug_test_outcome(sub.m.names[0], sub.m.names[1], sub.m.names + 2,
          sub.m.ncols - 2, pvalue, a);

      if (pvalue > a) {

        retval = ast_result_make(pvalue, min_pvalue, max_pvalue, TRUE,
                   sub.m.names + 2, sub.m.ncols - 2);

        Free1D(subset);
        FreeCOV(cov);
        FreeTAB(sub);
        Free1D(mean);
        Free1D(missing_xy);
        Free1D(missing_all);

        return retval;

      }/*THEN*/

    } while (next_subset(subset + nf + 2, cursize, dt.m.ncols - nf - 2, nf + 2));

    FreeCOV(cov);
    Free1D(subset);
    Free1D(mean);
    Free1D(missing_all);

  }/*FOR*/

  Free1D(missing_xy);
  FreeTAB(sub);

  return ast_result_make(pvalue, min_pvalue, max_pvalue, FALSE, NULL, 0);

}/*AST_GAUSTESTS_WITH_MISSING*/

/* parametric tests for Gaussian variables; the complete- and incomplete-data
 * paths differ too much to share a body, so dispatch on the tabular flags. */
ast_result ast_gaustests(tabular dt, int nf, int minsize, int maxsize, double a,
    bool debugging, test_e test) {

ast_result retval;

  if (tabular_is_complete(dt)) {

    /* the complete-data path reads the means (subset into each conditioning
     * set), so cache them here and free them afterwards. */
    tabular_cache_means(&dt, 0);
    retval = ast_gaustests_complete(dt, nf, minsize, maxsize, a, debugging,
               test);
    Free1D(dt.mean);

  }/*THEN*/
  else
    retval = ast_gaustests_with_missing(dt, nf, minsize, maxsize, a, debugging,
               test);

  return retval;

}/*AST_GAUSTESTS*/

/* conditional linear Gaussian test (for complete data). */
static ast_result ast_micg_complete(tabular dtx, tabular dty, tabular dtz,
    int nf, int minsize, int maxsize, double a, bool debugging) {

int i = 0, *subset = NULL, cursize = 0;
int *zptr = NULL, *zbuf = NULL, llx = 0, lly = 0, llz = 0;
double statistic = 0, pvalue = 0, min_pvalue = 1, max_pvalue = 0, df = 0;
void *xptr = 0, *yptr = 0;
ast_result retval;
tabular sub = { 0 };

  /* allocate a second data table to hold the conditioning variables. */
  sub = empty_tabular(dtz.m.nobs, dtz.ndcols, dtz.nccols);
  /* the discrete parents' configuration buffer is the same size for every
   * subset, so allocate it once and reuse it across the whole search. */
  zbuf = Calloc1D(dtz.m.nobs, sizeof(int));

  /* if both variables are continuous and all conditioning variables are
   * continuous, the test reverts back to a Gaussian mutual information test. */
  if (dtx.m.flag[0].discrete && dty.m.flag[0].discrete) {

    xptr = dtx.dcol[0];
    llx = dtx.nlvl[0];
    yptr = dty.dcol[0];
    lly = dty.nlvl[0];

  }/*THEN*/
  else if (dtx.m.flag[0].continuous && dty.m.flag[0].continuous) {

    xptr = dtx.ccol[0];
    yptr = dty.ccol[0];

  }/*THEN*/
  else if (dtx.m.flag[0].continuous && dty.m.flag[0].discrete) {

    xptr = dtx.ccol[0];
    yptr = dty.dcol[0];
    lly = dty.nlvl[0];

  }/*THEN*/
  else if (dtx.m.flag[0].discrete && dty.m.flag[0].continuous) {

    yptr = dtx.dcol[0];
    lly = dtx.nlvl[0];
    xptr = dty.ccol[0];

  }/*THEN*/

  for (cursize = imax(1, minsize); cursize <= maxsize; cursize++) {

    /* allocate and initialize the subset. */
    subset = Calloc1D(cursize + nf + 2, sizeof(int));
    /* initialize the first subset. */
    first_subset(subset + nf + 2, cursize, nf + 2);
    for (i = 0; i < nf + 2; i++)
      subset[i] = i;

    do {

      /* prepare the current subset. */
      tabular_subset_columns(&dtz, &sub, subset, cursize + nf + 2);

      /* if there are discrete conditioning variables, compute their
       * configurations. */
      if (sub.ndcols - 1 > 0) {

        zptr = zbuf;
        c_fast_config(sub.dcol + 1, sub.m.nobs, sub.ndcols - 1, sub.nlvl + 1,
          zptr, &llz, 1);

      }/*THEN*/
      else {

        zptr = NULL;
        llz = 0;

      }/*ELSE*/

      if (dtx.m.flag[0].discrete && dty.m.flag[0].discrete) {

        /* check whether the conditioning set is valid. */
        if (sub.nccols - 1 > 0) {

          /* need to reverse conditioning to actually compute the test. */
          statistic = 2 * sub.m.nobs *
                        c_cmicg_unroll(xptr, llx, yptr, lly, zptr, llz,
                          sub.ccol + 1, sub.nccols - 1, &df, sub.m.nobs);

        }/*THEN*/
        else {

          /* if both nodes are discrete, the test reverts back to a discrete
           * mutual information test. */
          statistic = c_cchisqtest(xptr, llx, yptr, lly, zptr, llz, sub.m.nobs,
                        &df, MI, TRUE);

        }/*ELSE*/

      }/*THEN*/
      else if (dtx.m.flag[0].continuous && dty.m.flag[0].continuous) {

        sub.ccol[0] = xptr;
        statistic = 2 * sub.m.nobs *
                      c_cmicg(yptr, sub.ccol, sub.nccols, NULL, 0, zptr, llz,
                        sub.nlvl, sub.m.nobs, &df);

      }/*THEN*/
      else { /* one variable is discrete, the other is continuous. */

        sub.dcol[0] = yptr;
        sub.nlvl[0] = lly;
        statistic = 2 * sub.m.nobs *
          c_cmicg(xptr, sub.ccol + 1, sub.nccols - 1, sub.dcol, sub.ndcols,
            zptr, llz, sub.nlvl, sub.m.nobs, &df);

      }/*ELSE*/

      pvalue = pchisq(statistic, df, FALSE, FALSE);
      update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      /* increment the test counter. */
      test_counter++;

      if (debugging)
        debug_test_outcome(dtx.m.names[0], dty.m.names[0], sub.m.names + 2,
          sub.m.ncols - 2, pvalue, a);

      if (pvalue > a) {

        retval = ast_result_make(pvalue, min_pvalue, max_pvalue, TRUE,
                   sub.m.names + 2, sub.m.ncols - 2);

        Free1D(subset);
        Free1D(zbuf);
        FreeTAB(sub);

        return retval;

      }/*THEN*/

    } while (next_subset(subset + nf + 2, cursize, dtz.m.ncols - nf - 2, nf + 2));

    Free1D(subset);

  }/*FOR*/

  Free1D(zbuf);
  FreeTAB(sub);

  return ast_result_make(pvalue, min_pvalue, max_pvalue, FALSE, NULL, 0);

}/*AST_MICG_COMPLETE*/

/* conditional linear Gaussian test (for incomplete data). */
static ast_result ast_micg_with_missing(tabular dtx, tabular dty, tabular dtz,
    int nf, int minsize, int maxsize, double a, bool debugging) {

int i = 0, *subset = NULL, cursize = 0;
int *zptr = NULL, *zbuf = NULL, llz = 0;
double statistic = 0, pvalue = 0, min_pvalue = 1, max_pvalue = 0, df = 0;
bool *missing_xy = NULL, *missing_all = NULL;
/* x and y are fixed across the search; a latent one makes every subset
 * degenerate. */
bool xy_latent = tabular_has_latent(dtx) || tabular_has_latent(dty);
ast_result retval;
tabular sub = { 0 }, sub_complete = { 0 };
tabular dtx_complete = { 0 }, dty_complete = { 0 };

  /* allocate a second data table to hold the conditioning variables. */
  sub = empty_tabular(dtz.m.nobs, dtz.ndcols, dtz.nccols);
  sub_complete = new_tabular(dtz.m.nobs, dtz.ndcols, dtz.nccols);
  sub_complete.m.names = Calloc1D(sub_complete.m.ncols, sizeof(char *));
  dtx_complete = new_tabular(dtx.m.nobs, dtx.ndcols, dtx.nccols);
  dty_complete = new_tabular(dty.m.nobs, dty.ndcols, dty.nccols);

  /* allocate missingness indicators. */
  missing_xy = Calloc1D(dtz.m.nobs, sizeof(bool));
  missing_all = Calloc1D(dtz.m.nobs, sizeof(bool));
  /* the discrete parents' configuration buffer never exceeds the number of
   * observations, so allocate it once and reuse it across the whole search. */
  zbuf = Calloc1D(dtz.m.nobs, sizeof(int));

  tabular_incomplete_cases(&dtx, missing_xy, 0, 0);
  tabular_incomplete_cases(&dty, missing_xy, 0, 0);

  for (cursize = imax(1, minsize); cursize <= maxsize; cursize++) {

    /* allocate and initialize the subset. */
    subset = Calloc1D(cursize + nf + 2, sizeof(int));
    /* initialize the first subset. */
    first_subset(subset + nf + 2, cursize, nf + 2);
    for (i = 0; i < nf + 2; i++)
      subset[i] = i;

    do {

      /* extract the variables in the conditioning set ... */
      tabular_subset_columns(&dtz, &sub, subset, cursize + nf + 2);

      if (xy_latent || tabular_has_latent(sub)) {

        /* a latent column leaves no complete cases; reproduce the empty
         * subsample's metadata (names and column count) so the degenerate
         * handling below reports the same conditioning set, and skip the scan
         * and subsampling. */
        for (int c = 0; c < sub.m.ncols; c++)
          sub_complete.m.names[c] = sub.m.names[c];
        sub_complete.m.ncols = sub.m.ncols;
        sub_complete.m.nobs = 0;

      }/*THEN*/
      else {

        /* ... find out which observations are missing ... */
        memcpy(missing_all, missing_xy, dtz.m.nobs * sizeof(bool));
        tabular_incomplete_cases(&sub, missing_all, 1, 1);
        /*  ... and subset the complete data. */
        tabular_subsample_by_logical(&sub, &sub_complete, missing_all, 1, 1);
        tabular_subsample_by_logical(&dtx, &dtx_complete, missing_all, 0, 0);
        tabular_subsample_by_logical(&dty, &dty_complete, missing_all, 0, 0);

      }/*ELSE*/

      /* assume independence and return if there are no complete observations. */
      if (sub_complete.m.nobs <= 1) {

        pvalue = min_pvalue = max_pvalue = 1;
        goto exit;

      }/*THEN*/

      /* if there are discrete conditioning variables, compute their
       * configurations. */
      if (sub_complete.ndcols - 1 > 0) {

        zptr = zbuf;
        c_fast_config(sub_complete.dcol + 1, sub_complete.m.nobs,
          sub_complete.ndcols - 1, sub_complete.nlvl + 1, zptr, &llz, 1);

      }/*THEN*/
      else {

        zptr = NULL;
        llz = 0;

      }/*ELSE*/

      if (dtx.m.flag[0].discrete && dty.m.flag[0].discrete) {

        /* check whether the conditioning set is valid. */
        if (sub_complete.nccols - 1 > 0) {

          /* need to reverse conditioning to actually compute the test. */
          statistic = 2 * sub_complete.m.nobs *
                        c_cmicg_unroll(dtx_complete.dcol[0],
                          dtx_complete.nlvl[0], dty_complete.dcol[0],
                          dty_complete.nlvl[0], zptr, llz, sub_complete.ccol + 1,
                          sub_complete.nccols - 1, &df, sub_complete.m.nobs);

        }/*THEN*/
        else {

          /* if both nodes are discrete, the test reverts back to a discrete
           * mutual information test. */
          statistic = c_cchisqtest(dtx_complete.dcol[0], dtx_complete.nlvl[0],
                        dty_complete.dcol[0], dty_complete.nlvl[0], zptr, llz,
                        sub_complete.m.nobs, &df, MI, TRUE);

        }/*ELSE*/

      }/*THEN*/
      else if (dtx.m.flag[0].continuous && dty.m.flag[0].continuous) {

        memcpy(sub_complete.ccol[0], dtx_complete.ccol[0],
          sub_complete.m.nobs * sizeof(double));
        statistic = 2 * sub_complete.m.nobs *
                      c_cmicg(dty_complete.ccol[0], sub_complete.ccol,
                        sub_complete.nccols, NULL, 0, zptr, llz,
                        sub_complete.nlvl, sub_complete.m.nobs, &df);

      }/*THEN*/
      else if (dtx.m.flag[0].continuous && dty.m.flag[0].discrete) {

        memcpy(sub_complete.dcol[0], dty_complete.dcol[0],
          sub_complete.m.nobs * sizeof(int));
        sub_complete.nlvl[0] = dty_complete.nlvl[0];
        statistic = 2 * sub_complete.m.nobs *
          c_cmicg(dtx_complete.ccol[0], sub_complete.ccol + 1,
            sub_complete.nccols - 1, sub_complete.dcol, sub_complete.ndcols,
            zptr, llz, sub_complete.nlvl, sub_complete.m.nobs, &df);

      }/*THEN*/
      else if (dtx.m.flag[0].discrete && dty.m.flag[0].continuous) {

        memcpy(sub_complete.dcol[0], dtx_complete.dcol[0],
          sub_complete.m.nobs * sizeof(int));
        sub_complete.nlvl[0] = dtx_complete.nlvl[0];
        statistic = 2 * sub_complete.m.nobs *
          c_cmicg(dty_complete.ccol[0], sub_complete.ccol + 1,
            sub_complete.nccols - 1, sub_complete.dcol, sub_complete.ndcols,
            zptr, llz, sub_complete.nlvl, sub_complete.m.nobs, &df);

      }/*ELSE*/

      pvalue = pchisq(statistic, df, FALSE, FALSE);
      update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      /* increment the test counter. */
      test_counter++;

exit:

      if (debugging)
        debug_test_outcome(dtx.m.names[0], dty.m.names[0],
          sub_complete.m.names + 2, sub_complete.m.ncols - 2, pvalue, a);

      if (pvalue > a) {

        retval = ast_result_make(pvalue, min_pvalue, max_pvalue, TRUE,
                   sub_complete.m.names + 2, sub_complete.m.ncols - 2);

        Free1D(subset);
        Free1D(zbuf);
        FreeTAB(sub);
        FreeTAB(sub_complete);
        FreeTAB(dtx_complete);
        FreeTAB(dty_complete);
        Free1D(missing_xy);
        Free1D(missing_all);

        return retval;

      }/*THEN*/

    } while (next_subset(subset + nf + 2, cursize, dtz.m.ncols - nf - 2, nf + 2));

    Free1D(subset);

  }/*FOR*/

  Free1D(zbuf);
  FreeTAB(sub);
  FreeTAB(sub_complete);
  FreeTAB(dtx_complete);
  FreeTAB(dty_complete);
  Free1D(missing_xy);
  Free1D(missing_all);

  return ast_result_make(pvalue, min_pvalue, max_pvalue, FALSE, NULL, 0);

}/*AST_MICG_WITH_MISSING*/

/* conditional linear Gaussian test; the complete- and incomplete-data paths
 * differ too much to share a body, so dispatch on the tabular flags. */
ast_result ast_micg(tabular dtx, tabular dty, tabular dtz, int nf, int minsize,
    int maxsize, double a, bool debugging) {

  if (tabular_is_complete(dtx) && tabular_is_complete(dty) &&
      tabular_is_complete(dtz))
    return ast_micg_complete(dtx, dty, dtz, nf, minsize, maxsize, a, debugging);
  else
    return ast_micg_with_missing(dtx, dty, dtz, nf, minsize, maxsize, a,
             debugging);

}/*AST_MICG*/

/* discrete permutation tests. */
ast_result ast_dperm(tabular dtx, tabular dty, tabular dtz, int nf, int minsize,
    int maxsize, double a, test_e type, int nperms, double threshold,
    bool debugging) {

int *xptr = dtx.dcol[0], *yptr = dty.dcol[0], *zptr = NULL, *subset = NULL;
int i = 0, cursize = 0, llx = dtx.nlvl[0], lly = dty.nlvl[0], llz = 0;
double statistic = 0, pvalue = 0, min_pvalue = 1, max_pvalue = 0, df = 0;
bool xy_latent = tabular_has_latent(dtx) || tabular_has_latent(dty);
ast_result retval;
tabular sub = { 0 };

  /* allocate a second data table to hold the conditioning variables. */
  sub = empty_tabular(dtz.m.nobs, dtz.m.ncols, 0);
  /* allocate the parents' configurations. */
  zptr = Calloc1D(dtz.m.nobs, sizeof(int));

  for (cursize = imax(1, minsize); cursize <= maxsize; cursize++) {

    /* allocate and initialize the subset. */
    subset = Calloc1D(cursize + nf, sizeof(int));
    /* initialize the first subset. */
    first_subset(subset + nf, cursize, nf);
    for (i = 0; i < nf; i++)
      subset[i] = i;

    /* iterate over subsets. */
    do {

      /* prepare the current subset. */
      tabular_subset_columns(&dtz, &sub, subset, cursize + nf);

      if (xy_latent || tabular_has_latent(sub)) {

        /* a latent variable makes the contingency table empty: the permutation
         * test is degenerate (and consumes no random numbers). */
        pvalue = 1;
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/
      else {

      /* construct the parents' configurations. */
      c_fast_config(sub.dcol, sub.m.nobs, cursize + nf, sub.nlvl, zptr, &llz, 1);

      c_cmcarlo(xptr, llx, yptr, lly, zptr, llz, sub.m.nobs, nperms, &statistic,
        &pvalue, threshold, type, &df);
      update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*ELSE*/

      /* increment the test counter. */
      test_counter++;

      if (debugging)
        debug_test_outcome(dtx.m.names[0], dty.m.names[0], sub.m.names,
          sub.m.ncols, pvalue, a);

      if (pvalue > a) {

        retval = ast_result_make(pvalue, min_pvalue, max_pvalue, TRUE,
                   sub.m.names, sub.m.ncols);

        Free1D(subset);
        Free1D(zptr);
        FreeTAB(sub);

        return retval;

      }/*THEN*/

    } while (next_subset(subset + nf, cursize, dtz.m.ncols - nf, nf));

    Free1D(subset);

  }/*FOR*/

  Free1D(zptr);
  FreeTAB(sub);

  return ast_result_make(pvalue, min_pvalue, max_pvalue, FALSE, NULL, 0);

}/*AST_DPERM*/

/* continuous permutation tests. */
ast_result ast_gperm(tabular dt, int nf, int minsize, int maxsize, double a,
    test_e type, int nperms, double threshold, bool complete, bool debugging) {

int i = 0, j = 0, k = 0, cursize = 0, nc = 0, *subset = NULL;
double statistic = 0, pvalue = 0, min_pvalue = 1, max_pvalue = 0;
double **complete_column = NULL;
bool *missing_xy = NULL, *missing_z = NULL;
ast_result retval;
tabular sub = { 0 }, sub_complete = { 0 };

  /* allocate a second data table to hold the conditioning variables. */
  sub = empty_tabular(dt.m.nobs, 0, dt.m.ncols);
  sub.mean = Calloc1D(dt.m.ncols, sizeof(double));

  if (!complete) {

    missing_xy = Calloc1D(dt.m.nobs, sizeof(bool));
    missing_z = Calloc1D(dt.m.nobs, sizeof(bool));

    tabular_incomplete_cases_range(&dt, missing_xy, 0, -1, 0, 1);

  }/*THEN*/

  for (cursize = imax(1, minsize); cursize <= maxsize; cursize++) {

    /* allocate and initialize the subset indexes array. */
    subset = Calloc1D(cursize + nf + 2, sizeof(int));
    /* initialize the first subset. */
    first_subset(subset + nf + 2, cursize, nf + 2);
    for (i = 0; i < nf + 2; i++)
      subset[i] = i;

    if (!complete)
      sub_complete = new_tabular(dt.m.nobs, 0, cursize + nf + 2);

    /* iterate over subsets. */
    do {

      /* prepare the current subset. */
      tabular_subset_columns(&dt, &sub, subset, cursize + nf + 2);

      if (tabular_has_latent(sub)) {

        /* a latent variable leaves no complete observations: the permutation
         * test is degenerate (and consumes no random numbers). */
        pvalue = 1;
        update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*THEN*/
      else {

      if (!complete) {

        memset(missing_z, '\0', sizeof(bool) * sub.m.nobs);
        tabular_incomplete_cases(&sub, missing_z, 0, 2);
        complete_column = sub_complete.ccol;

        for (k = 0, nc = 0; k < sub.m.nobs; k++) {

          if (missing_xy[k] || missing_z[k])
            continue;

          for (j = 0; j < sub.m.ncols; j++)
            complete_column[j][nc] = sub.ccol[j][k];
          nc++;

        }/*FOR*/

      }/*THEN*/
      else {

        complete_column = sub.ccol;
        nc = sub.m.nobs;

      }/*ELSE*/

      c_gauss_cmcarlo(complete_column, sub.m.ncols, nc, 0, 1, nperms,
        &statistic, &pvalue, threshold, type);
      update_pvalue_range(pvalue, &min_pvalue, &max_pvalue);

      }/*ELSE*/

      /* increment the test counter. */
      test_counter++;

      if (debugging)
        debug_test_outcome(sub.m.names[0], sub.m.names[1], sub.m.names + 2,
          sub.m.ncols - 2, pvalue, a);

      if (pvalue > a) {

        retval = ast_result_make(pvalue, min_pvalue, max_pvalue, TRUE,
                   sub.m.names + 2, sub.m.ncols - 2);

        if (!complete) {

          Free1D(missing_xy);
          Free1D(missing_z);

        }/*THEN*/

        Free1D(subset);
        FreeTAB(sub);
        FreeTAB(sub_complete);

        return retval;

      }/*THEN*/

    } while (next_subset(subset + nf + 2, cursize, dt.m.ncols - nf - 2, nf + 2));

    FreeTAB(sub_complete);
    Free1D(subset);

  }/*FOR*/

  if (!complete) {

    Free1D(missing_xy);
    Free1D(missing_z);

  }/*THEN*/

  FreeTAB(sub);

  return ast_result_make(pvalue, min_pvalue, max_pvalue, FALSE, NULL, 0);

}/*AST_GPERM*/
