*=================================================================================================================================================
* Date : 2024-01-23
* Stata version : Stata 18 SE
*
* This program analyses simulated data accounting for DIF through a partial credit model
*
* ado-files needed : - pcm, rosali (version 5.5 October 25, 2023, available on gitea)
*
*
*================================================================================================================================================
adopath+"/home/corentin/Documents/These/Recherche/ROSALI-SIM/Modules/rosali_custom"


local N = "50 100 200 300"
 local ss = "20"
 foreach s in `ss' {
 foreach Nnn in `N' {
 local Nn =  `Nnn'
 local path_data = "/home/corentin/Documents/These/Recherche/Simulations/Data/DIF/N`Nn'"
 if (`s'<=4) {
   local path_data = "/home/corentin/Documents/These/Recherche/Simulations/Data/NoDIF/N`Nn'"
 }
 local path_res = "/home/corentin/Documents/These/Recherche/Simulations/Analysis/ROSALI-DIF/N`Nn'"
 local scenarios = "A B C D E F G"
 foreach scen in `scenarios' {
 clear
 import delim "`path_data'/scenario_`s'`scen'_`Nn'.csv", encoding(ISO-8859-2)  case(preserve) clear
 rename TT tt

  if (`s'<=2) {
    local nbitems=4
  }
  else if (`s'<=4) {
    local nbitems=7
  }
  else if (`s'<=12) {
    local nbitems=4
  }
  else {
    local nbitems=7
  }

 if (mod(`s',2)==0) {
   local nbmoda=3
 }
 else {
   local nbmoda=1
 }

  if (`s'<=4) {
    local nbdif=0
  }
  else if (`s'<=8) {
    local nbdif=1
  }
  else if (`s'<=16) {
    local nbdif=2
  }
  else {
    local nbdif=3
  }
  * taillemat = Maximum J*M cases pour les items par et J*M cases pour les dif par + J cases pour les DIF detect + nbdif cases pour dif réel
  local taillemat=2*`nbitems'*`nbmoda'+`nbitems'+`nbdif'+2
  if (mod(`s',2)==0) {
    local taillemat=2*`nbitems'*`nbmoda'+`nbitems'+`nbitems'+`nbdif'+2
  }
  local colna=""
  forvalues i=1/`nbitems' {
    forvalues z=1/`nbmoda' {
      local colna = "`colna'"+"item`i'_`z' "
      local colna = "`colna'"+"dif_`i'_`z' "
    }
  }
  forvalues i=1/`nbitems' {
    if (mod(`s',2)==1) {
      local colna = "`colna'"+"dif_detect_`i' "
    }
    if (mod(`s',2)==0) {
      local colna = "`colna'"+"dif_detect_`i' "+"dif_detect_unif_`i' "
    }
  }

  forvalues i=1/`nbdif' {
    local colna = "`colna'"+"real_dif_`i' "
  }
 local colna = "`colna'" + "beta " + "se_beta"


  mat outmat = J(1000,`taillemat',.)
  mat colnames outmat= `colna'
  di "Scenario `s'`scen' / N=`Nnn'"
  forvalues k=1/1000 {
    if (mod(`k',100)==0) {
      di "`k'/1000"
    }
    preserve
    qui keep if replication==`k'


    * MERGE des modalités si non représentées

    if (`nbmoda'>1 & `Nn'==50) {
          local com_z = 0
          qui gen comz = 0
          forvalues j = 1 / `nbitems' {
          local recoda_`j' = 0
          qui tab item`j' if tt == 0, matrow(rect1_g0_`j') matcell(nbrt1_g0_`j')
          local maxm`j'_t1_g0 = rect1_g0_`j'[r(r),1]
          local minm`j'_t1_g0 = rect1_g0_`j'[1,1]

          qui tab item`j' if tt == 1, matrow(rect1_g1_`j') matcell(nbrt1_g1_`j')
          local minm`j'_t1_g1 = rect1_g1_`j'[1,1]
          local maxm`j'_t1_g1 = rect1_g1_`j'[r(r),1]

          local minm_`j' = min(`minm`j'_t1_g0',`minm`j'_t1_g1')
          local maxm_`j' = max(`maxm`j'_t1_g0',`maxm`j'_t1_g1')
          local nbm_`j' =  `=`maxm_`j''-`minm_`j'''

          if `minm_`j'' != 0 & `com_z' == 0 {
            local com_z = 1
          }

          qui count if item`j' == 3 & tt == 0
          local mod3plac = r(N)
          qui count if item`j' == 3 & tt == 1
          local mod3tt = r(N)
          local nb_rn3 = min(`mod3plac',`mod3tt')
          if `nb_rn3'==0 {
            qui replace comz = 1
          }

          forvalues m = 0/`=`nbm_`j''-1' {
            qui count if item`j' == `m' & tt == 0
            local nb_rn1_g0 = r(N)
            qui count if item`j' == `m' & tt == 1
            local nb_rn1_g1 = r(N)
            local nb_rn = min(`nb_rn1_g0',`nb_rn1_g1')
            if `nb_rn' == 0 {
              qui replace comz = 1
              local recoda_`j' = 1
              if  `m' == 0 | `m' < `minm`j'_t1_g0' | `m' < `minm`j'_t1_g1' {
                local stop = 1
                forvalues kk = 1/`=`nbm_`j''-`m'' {
                  qui count if item`j' == `=`m' + `kk'' & tt == 0
                  local v`kk'1_0 = r(N)
                  qui count if item`j' == `=`m' + `kk'' & tt == 1
                  local v`kk'1_1 = r(N)
                  if (`v`kk'1_0' != 0 | `v`kk'1_1' != 0) & `stop' != 0 {
                    qui replace item`j'= `=`m'+`kk'' if item`j'==`m'
                    local zzz=`j'+`nbitems'
                    *qui replace item`zzz'=`=`m'+`kk'' if item``=`j'+`nbitems'''==`m'
                    *di  "WARNING SCENARIO `k': items `j': answers `m' and `=`m'+`kk'' merged"
                    local stop = 0
                  }
                }
              }
              else if `m' == `=`nbm_`j''-1' | `m' >= `maxm`j'_t1_g1' {
                local stop = 1
                forvalues kk = 1/`=`m'' {
                  qui count if item`j' == `=`m' - `kk'' & tt == 0
                  local v`kk'1_0 = r(N)
                  qui count if item`j' == `=`m' - `kk'' & tt == 1
                  local v`kk'1_1 = r(N)
                  if (`v`kk'1_0' != 0 | `v`kk'1_1' != 0) & `stop' != 0 {
                    qui replace item`j'= `=`m' - `kk'' if item`j'==`m'
                    local zzz=`j'+`nbitems'
                    *qui replace item`zzz'= `=`m' - `kk'' if item`zzz'==`m'
                    *di  "WARNING SCENARIO `k': items `j': answers `m' and `=`m'+`kk'' merged"
                    local stop = 0
                  }
                }
              }
              else {
                if runiform()>0.5{
                  local stop = 1
                  forvalues kk = 1/`m' {
                    qui count if item`j' == `=`m' - `kk'' & tt == 0
                    local v`kk'1_0 = r(N)
                    qui count if item`j' == `=`m' - `kk'' & tt == 1
                    local v`kk'1_1 = r(N)
                    if (`v`kk'1_0' != 0 | `v`kk'1_1' != 0) & `stop' != 0 {
                      qui replace item`j'= `=`m'-`kk'' if item`j'==`m'
                      local zzz=`j'+`nbitems'
                      *qui replace item`zzz'=`=`m'-`kk'' if item``=`j'+`nbitems'''==`m'
                      *di  "WARNING SCENARIO `k': items `j': answers `m' and `=`m'+`kk'' merged"
                      local stop = 0
                    }
                  }
                }
                else {
                  local stop = 1
                  forvalues kk = 1/`=`nbm_`j''-`m'' {
                    qui count if item`j' == `=`m' + `kk'' & tt == 0
                    local v`kk'1_0 = r(N)
                    qui count if item`j' == `=`m' + `kk'' & tt == 1
                    local v`kk'1_1 = r(N)
                    if (`v`kk'1_0' != 0 | `v`kk'1_1' != 0) & `stop' != 0{
                      qui replace item`j'=`=`m' + `kk'' if item`j'==`m'
                      local zzz=`j'+`nbitems'
                      *qui replace item`zzz'=`=`m' + `kk'' if item``=`j'+`nbitems'''==`m'
                      *di  "WARNING SCENARIO `k': items `j': answers `m' and `=`m'+`kk'' merged"
                      local stop = 0
                    }
                    else {
                      if `stop' != 0 {
                        qui replace item`j'= `nbm_`j'' if item`j'==`m'
                        local zzz=`j'+`nbitems'
                        *qui replace item`zzz'= `nbm_`j'' if item``=`j'+`nbitems'''==`m'
                        *di  "WARNING SCENARIO `k': items `j': answers `m' and `=`m'+`kk'' merged"
                        local stop = 0
                      }
                    }
                  }
                }
              }
            }
          }
          qui levelsof item`j'
          local val = r(levels)
          local checker: word 1 of `val'
          local checker2: word 2 of `val'
          local checker3: word 3 of `val'
          local nummoda=r(r)
          local nbmoda_`j'=`nummoda'-1
          if (`nummoda'==2) {
            qui recode item`j' (`checker'=0) (`checker2'=1)
          }
          if (`nummoda'==3) {
          if (`checker'!=0) {
            qui recode item`j' (`checker'=0) (`checker2'=1) (`checker3'=2)
          }
          else if (`checker2'!=1) {
            qui recode item`j' (`checker2'=1) (`checker3'=2)
          }
          else if (`checker3'!=2) {
            qui recode item`j' (`checker3'=2)
          }
        }
      }

      qui valuesof comz
      local val = r(values)
      local checker: word 1 of `val'
    }
    else {
      forvalues jj=1/`nbitems' {
        local nbmoda_`jj'=`nbmoda'
      }
    }


    * ROSALI
    qui rosali_original item1-item`nbitems' item1-item`nbitems', group(tt)
    qui mat resmat=r(difitems)
    local nbitems2 = 2*`nbitems'

    * Calculer le nbre d'items détectés
    local nbdetect = 0
    local stop = 0
    forvalues jj=1/`nbitems' {
        if (`stop'==0) {
          mat testm=J(1,1,.)
          if (resmat[1,`jj']==testm[1,1]) {
            local stop = 1
            local nbdetect = `jj'-1
          }
        }
    }

    * Stocker les items détectés +
    * Définition des contraintes
    local csrt=0
    mat testm=J(1,1,0)
    forvalues u=1/`nbdetect' {
       local difitems`u'=resmat[1,`u']
        local i=`difitems`u''
         if (`nbmoda_`i''==3 & resmat[1,`nbitems'+`i']!=testm[1,1]){
          local constrnt`u' = "constraint `u' 2*([1.item`i']_cons-([1.item`i']_cons+[1.item`i'] tt))=([2.item`i']_cons-([2.item`i']_cons+[2.item`i'] tt))"
          qui `constrnt`u''
          local v=`u'+100
          local constrnt`u'_2 = "constraint `v' 3*([1.item`i']_cons-([1.item`i']_cons+[1.item`i'] tt))=([3.item`i']_cons-([3.item`i']_cons+[3.item`i'] tt))"
          qui `constrnt`u'_2'
         }
         if (`nbmoda_`i''==2 & resmat[1,`nbitems'+`i']!=testm[1,1]){
          local constrnt`u' = "constraint `u' 2*([1.item`i']_cons-([1.item`i']_cons+[1.item`i'] tt))=([2.item`i']_cons-([2.item`i']_cons+[2.item`i'] tt))"
          qui `constrnt`u''
         }
        }

    * Définition du modèle
    local mod "gsem "
    local conformula = ""
    forvalues i=1/`nbitems' {
        local mod = "`mod'"+"(1.item`i'<-THETA@1)"
        if (`nbmoda_`i''==3) {
          local mod = "`mod'"+"(2.item`i'<-THETA@2)(3.item`i'<-THETA@3)"
        }
        else if (`nbmoda_`i''==2) {
          local mod = "`mod'"+"(2.item`i'<-THETA@2)"
        }
    }
    forvalues u=1/`nbdetect' {
        local v=`difitems`u''
        local mod = "`mod'"+"(1.item`v'<-THETA@1 tt)"
        if (`nbmoda_`v''==3) {
          local mod = "`mod'"+"(2.item`v'<-THETA@2 tt)(3.item`v'<-THETA@3 tt)"
        }
        else if (`nbmoda_`v''==2) {
          local mod = "`mod'"+"(2.item`v'<-THETA@2 tt)"
        }
        local w= 100+`u'
        local unif_`u'=0
        if (resmat[1,`nbitems'+`v']!=testm[1,1] & `nbmoda_`v''==3) {
          local conformula = "`conformula'" + "`u' " + "`w' "
          local unif_`u'=1
        }
        else if (resmat[1,`nbitems'+`v']!=testm[1,1] & `nbmoda_`v''==2) {
          local conformula = "`conformula'" + "`u' "
          local unif_`u'=1
        }
    }
    if ("`conformula'" != "") {
      local mod = "`mod'" + "(THETA<-tt), mlogit tol(0.01) iterate(500) latent(THETA) nocapslatent constraint(`conformula')"
    }
    else {
      local mod = "`mod'" + "(THETA<-tt), mlogit tol(0.01) iterate(500) latent(THETA) nocapslatent"
    }
    *calcul du modèle
    qui `mod'
    mat V=r(table)
    mat W=V[1..2,1...]

    * compilation
    forvalues j=1/`nbitems' {
      forvalues z=1/`nbmoda_`j'' {
        mat outmat[`k',colnumb(outmat,"item`j'_`z'")] = W[1,colnumb(W,"`z'.item`j':_cons")]
      }
    }
    * compilation DIF
    forvalues u=1/`nbdetect' {
      local j=`difitems`u''
        forvalues z=1/`nbmoda_`j'' {
          mat outmat[`k',colnumb(outmat,"dif_`u'_`z'")] = W[1,colnumb(W,"`z'.item`j':tt")]
        }
        mat outmat[`k',colnumb(outmat,"dif_detect_`u'")] = `j'
        if (mod(`s',2)==0) {
          mat outmat[`k',colnumb(outmat,"dif_detect_unif_`u'")] = `unif_`u''
        }
      }

    * Stocker les items de DIF originaux
    if (`nbdif' > 0) {
      qui levelsof dif1
      local ldif1 = r(levels)
      local diff1: word 1 of `ldif1'
      qui mat outmat[`k',colnumb(outmat,"real_dif_1")]=`diff1'
      if (`nbdif' > 1) {
        qui levelsof dif2
        local ldif2 = r(levels)
        local diff2: word 1 of `ldif2'
        qui mat outmat[`k',colnumb(outmat,"real_dif_2")]=`diff2'
        if (`nbdif' > 2) {
          qui levelsof dif3
          local ldif3 = r(levels)
          local diff3: word 1 of `ldif3'
          qui mat outmat[`k',colnumb(outmat,"real_dif_3")]=`diff3'
        }
      }
    }
    qui mat outmat[`k',colnumb(outmat,"beta")]=W[1,colnumb(W,"THETA:tt")]
    qui mat outmat[`k',colnumb(outmat,"se_beta")]=W[2,colnumb(W,"THETA:tt")]
    restore
  }
  putexcel set "`path_res'/`s'`scen'_`Nn'_original.xls", sheet("outmat") replace
  putexcel A1=matrix(outmat), colnames
 }
 }
 }
