How to create very large ntuples (RWN or CWN) ?

Very large ntuples should be created "disk resident". By default CWN are created that way but RWN are not. An other FAQ explain how to do this for RWN. But creating the ntuple "disk resident" is not enough to make it "very large". If you get one of the following error message:

   RZOUT: current RZ file cannot support > 64K records
          or individual directories > 64K
   RZOUT: previous cycle(s) for this key (      30)  deleted
   RZOUT: please consult ZEBRA manual for further details

or:


 RZOUT: "Request exceeds quota"

this indicates that some parameters must be tuned in order to bypass these limits:

Increase the record length: CALL HROPEN with a large value of LRECL (ex LRECL=4096 instead of 1024) and you can get files 4 times bigger. When you do that, do not forget in PAW (in the command HI/FILE) to specify the LRECL parameter. Note that if the record length is smaller than 8192, the PAW command HI/FILE is able to determine the correct record length if the LRECL parameter is set to 0.
Increase the maximum number of records: If changing LRECL is not sufficient, you can also specify the option Q in CHOPT and in IQUEST(10) (COMMON/QUEST/IQUEST(100)) give the maximum number of records. The default value is 32000 records.
Example:
```
     COMMON/QUEST/IQUEST(100)
     IQUEST(10)=64000
     CALL HROPEN(lun,......,'NQ',4096,istat)
 
```
will create a file with a maximum of 64000 records of 4096 words. With this technique the largest number of records you can have in an HBOOK file will be 64000 (2**16).
Important note: It is very important to initialise IQUEST(10) just before the call to HROPEN. The QUEST common block is used a lot in the whole CERNLIB, and in particular in HBOOK, so if the value of IQUEST(10) is defined too far from the call to HROPEN, very likely it will not have the expected value when needed.

Increase the maximum number of records even more: If increasing the number of record to 2**16 is still not enough, you can use the option QE (E for "Extended") available in HBOOK since version 4.27/03. This option allows to have up to 2**32 records in an HBOOK file. The following example demonstrates how to use it:

       Program Quota
       Parameter (Nwpawc=150000, Nvars=17)
       Character*8 Chtags(Nvars)
       Dimension Event(Nvars)
       Common/Pawc/Paw(Nwpawc)
 *
       Equivalence (Event(1) ,X1)    , (Event(2),Y1)    , (Event(3) ,Z1)
       Equivalence (Event(4) ,Enrgy1), (Event(5),Eloss1)
       Equivalence (Event(6) ,X2)    , (Event(7),Y2)    , (Event(8) ,Z2)
       Equivalence (Event(9) ,Enrgy2), (Event(10),Eloss2)
       Equivalence (Event(11),X3)    , (Event(12),Y3)   , (Event(13),Z3)
       Equivalence (Event(14),Enrgy3), (Event(15),Eloss3)
       Equivalence (Event(16),X4)    , (Event(17),Y4)
 *
       Common /QUEST/ Iquest(100)
       Data Chtags/'X1','Y1','Z1','Energy1','Eloss1',
      +            'X2','Y2','Z2','Energy2','Eloss2',
      +            'X3','Y3','Z3','Energy3','Eloss3',
      +            'X4','Y4'/
 *
       Call Hlimit(Nwpawc)
 *
       Iquest(10) = 256000
       Call Hropen (1,'QUOTA','quota.hbook','NQE',1024,ISTAT)
       If (Istat.Ne.0) Stop
 *
       Call Hbookn (10,'A simple Ntuple',Nvars,'//QUOTA',10000,Chtags)
 *
       Do I=1, 5000000
          Call Rannor(X1,Y1)
          X1     = Float(I)
          Z1     = Sqrt(X1*X1+Y1*Y1)
          Enrgy1 = 50. + 10.*X1
          Eloss1 = 10. * Abs(Y1)
          Call Rannor (X2,Y2)
          Z2     = Sqrt(X2*X2+Y2*Y2)
          Enrgy2 = 50. + 10.*X2
          Eloss2 = 10. * Abs(Y2)
          Call Rannor(X3,Y3)
          Z3     = SQRT(X3*X3+Y3*Y3)
          Enrgy3 = 50. + 10.*X3
          Eloss3 = 10. * Abs(Y3)
          Call Rannor(X4,Y4)
          Call Hfn(10,Event)
       Enddo
       Call Hrout(0,Icycle,' ')
       Call Hrend('QUOTA')
       End

When you run this example you get:

 $ ./quota
  RZMAKE. new RZ format selected.
          This file will not be readable with versions of RZ prior to release 94B
 $ ls -l quota.hbook
 -rw-r--r--   1 couet    olivier  340918272 Dec 13 12:06 quota.hbook
 $

The C version of this program is:

 #include <stdlib.h>
 #include <stdio.h>
 #include <cfortran.h>
 #include <math.h>
 #include <packlib.h>
 #include <kernlib.h>

 #define Nwpawc 150000
 #define Nvars  17

 typedef float PAWC_DEF[Nwpawc];
 #define PAWC COMMON_BLOCK(PAWC,pawc)
 COMMON_BLOCK_DEF(PAWC_DEF,PAWC);
 
 typedef struct {int iquest[100];} quest_def;
 #define QUEST COMMON_BLOCK(QUEST,quest)
 COMMON_BLOCK_DEF(quest_def,QUEST);
 
 main()
 {
    int i, Icycle, istat;
    int record_size = 1024;

    float Event[Nvars];

    char Chtags[17][8]={"X1","Y1","Z1","Energy1","Eloss1",
                        "X2","Y2","Z2","Energy2","Eloss2",
                        "X3","Y3","Z3","Energy3","Eloss3",
                        "X4","Y4"};

    HLIMIT(Nwpawc);

    QUEST.iquest[9] = 256000;
    HROPEN(1,"QUOTA","quota.hbook","NQE",record_size,istat);

    HBOOKN(10,"A simple Ntuple",Nvars,"//QUOTA",10000,Chtags);

    for (i=1; i<=5000000; i++) {
       RANNOR(Event[0],Event[1]);
       Event[0] = (float)i;
       Event[2] = sqrt(Event[0]*Event[0]+Event[1]*Event[1]);
       Event[3] = 50. + 10.*Event[0];
       Event[4] = 10. * abs(Event[1]);
       RANNOR (Event[5],Event[6]);
       Event[7] = sqrt(Event[5]*Event[5]+Event[6]*Event[6]);
       Event[8] = 50. + 10.*Event[5];
       Event[9] = 10. * abs(Event[6]);
       RANNOR(Event[10],Event[11]);
       Event[12] = sqrt(Event[10]*Event[10]+Event[11]*Event[11]);
       Event[13] = 50. + 10.*Event[10];
       Event[14] = 10. * abs(Event[11]);
       RANNOR(Event[15],Event[16]);
       HFN(10,Event);
    }

    HROUT(0,Icycle," ");
    HREND("QUOTA");
    KUCLOS(1," ",1);
 }

To gain some records you can also try to minimise the number of keys in the HBOOK/RZ directory. To do that the ntuple buffer size can be increased:

If it is a row wise ntuple (RWN: created with HBOOKN): Set parameter NWPRIM to a large value (say 50000 words)
If it is a column wise ntuple (CWN: created with HBNT/HBNAME): Set the buffer size for each column via CALL HBSET('BSIZE',NWORDS,IERR). A convenient value for NWORDS is the record size itself (eg, 4096 words)