From oldham at codesourcery.com  Mon Dec  3 20:39:43 2001
From: oldham at codesourcery.com (Jeffrey Oldham)
Date: Mon, 3 Dec 2001 12:39:43 -0800
Subject: Patch: Fix Typos and Add Missing Semicolons
Message-ID: <20011203123943.A16508@codesourcery.com>

This patch fixes mainly typographical errors I found in the Pooma
source code.  Additionally, there are two more significant revisions:

1) Ended a statement with a semicolon in
   src/Tulip/CollectFromContexts.h.  I do not understand why this was
   not previously found.

2) Discovered a "hidden" paragraph in docs/Layout.html.*  Should this
   paragraph really be present?  Presumably, this documentation will
   soon disappear, but it would be nice to get it correct in case it
   does survives.

2001-11-29 Jeffrey D. Oldham  <oldham at codesourcery.com>

        * benchmarks/Doof2d/Doof2d.h: Fix typo in comment.
        (*::initializeStorage): Generalize comment to work for any container.
        * docs/Layout.html: Fix numerous typographical and speling errors.
        Add missing quotation mark which caused entire paragraph
        describing tags to not be displayed.
        * src/Engine/IndexFunctionEngine.h (Engine<Dim, T,
        IndexFunction<Functor> >): Fix typo in introductory comment.
        * src/Engine/NotifyEngineWrite.h: Refill comment.
        * src/Engine/UserFunction.h: Use correct article in comment.
        * src/Field/Mesh/UniformRectilinearMesh.h: End sentence with a
        period in comment.
        * src/Layout/UniformGridLayout.cpp
        (UniformGridLayoutData<Dim>::repartition): Fix typo in
        introductory comment.
        * src/Layout/UniformGridLayout.h (UniformGridLayout<Dim>): Remove
        extra comment characters.
        * src/Partition/DistributedMapper.h (DistributedMapper<Dim>::map):
        Capitalize beginning of sentence in comment.
        * src/Partition/GridPartition.h (GridPartition<Dim>): Add spaces
        between comment symbols and text.  Fix typos and spelling
        mistakes.
        * src/Partition/UniformGridPartition.h
        (UniformGridPartition<Dim>): Properly indent the code.
        * src/Tulip/CollectFromContexts.h (Serialize<CHEETAH,
        CollectionValue<T> >::size): Add missing semicolon at statement's
        end.
        * src/Utilities/Benchmark.h: Fix typo in introductory comment.
        * src/Utilities/Conform.h: Change word in introductory comment.
        * src/Utilities/ModelElement.h: Fix incorrect introductory comment.
        * src/Utilities/Statistics.h: Fix incorrect introductory comment.
        * src/Utilities/WrappedInt.h: Fix spelling mistake in introductory
        comment.

Tested on Linux with g++ 3.1 by compiling Pooma library and Doof2d benchmark
Approved by     Mark Mitchell
Applied to      mainline

Thanks,
Jeffrey D. Oldham
oldham at codesourcery.com
-------------- next part --------------
Index: benchmarks/Doof2d/Doof2d.h
===================================================================
RCS file: /home/pooma/Repository/r2/benchmarks/Doof2d/Doof2d.h,v
retrieving revision 1.3
diff -c -p -r1.3 Doof2d.h
*** benchmarks/Doof2d/Doof2d.h	2001/10/16 18:26:09	1.3
--- benchmarks/Doof2d/Doof2d.h	2001/11/30 04:21:07
***************
*** 28,34 ****
  
  //-----------------------------------------------------------------------------
  // Classes Doof2dCppTran, Doof2dP2, DoofNinePt, Doof2dOpt
! // Implementation Classes Doof2dStorage, Doof2dBase, 
  //-----------------------------------------------------------------------------
  
  #ifndef POOMA_BENCHMARKS_DOOF2D_H
--- 28,34 ----
  
  //-----------------------------------------------------------------------------
  // Classes Doof2dCppTran, Doof2dP2, DoofNinePt, Doof2dOpt
! // Implementation Classes Doof2dStorage, Doof2dBase
  //-----------------------------------------------------------------------------
  
  #ifndef POOMA_BENCHMARKS_DOOF2D_H
*************** public:
*** 119,125 ****
  
    void initializeStorage(int &n, int np, int ng) 
    {
!     // Get new array domain, including "guards".
      
      Interval<1> N(1, n);
      Interval<2> vertDomain(N, N);
--- 119,125 ----
  
    void initializeStorage(int &n, int np, int ng) 
    {
!     // Create the domain, including "guards".
      
      Interval<1> N(1, n);
      Interval<2> vertDomain(N, N);
*************** public:
*** 140,146 ****
  
    void initializeStorage(int &n, int np, int ng) 
    {
!     // Get new array domain, including "guards".
  
      Interval<1> N(1, n);
      Interval<2> vertDomain(N, N);
--- 140,146 ----
  
    void initializeStorage(int &n, int np, int ng) 
    {
!     // Create the domain, including "guards".
  
      Interval<1> N(1, n);
      Interval<2> vertDomain(N, N);
*************** public:
*** 169,175 ****
    {
      n = (n / np) * np;
      
!     // Get new array domain.
      
      Interval<1> N(1, n);
      Interval<2> newDomain(N, N);
--- 169,175 ----
    {
      n = (n / np) * np;
      
!     // Create the domain.
      
      Interval<1> N(1, n);
      Interval<2> newDomain(N, N);
*************** public:
*** 208,214 ****
    {
      n = (n / np) * np;
      
!     // Get new array domain.
      
      Interval<1> N(1, n);
      Interval<2> newDomain(N, N);
--- 208,214 ----
    {
      n = (n / np) * np;
      
!     // Create the domain.
      
      Interval<1> N(1, n);
      Interval<2> newDomain(N, N);
Index: docs/Layout.html
===================================================================
RCS file: /home/pooma/Repository/r2/docs/Layout.html,v
retrieving revision 1.1
diff -c -p -r1.1 Layout.html
*** docs/Layout.html	2001/03/19 16:11:13	1.1
--- docs/Layout.html	2001/11/30 04:22:44
*************** guard layer thicknesses. External guard 
*** 59,80 ****
  specified by the internal guard layer for those patches that make up the
  edges of the data object.
  <p>There are two types of GuardLayers used by a MultiPatchEngine based
! Pooma data object. These are internal guard Layers, and external guard
! layers. Each data patch that makes up POOMA data object has it's domain
! extended by the number of data elements specified by the GuardLayers object.&nbsp;
! Internal guard layers are layers of elements or cells added to the upper
  and/or lower end of the sub-domain of the PatchEngine.&nbsp; These cells
  are to be filled with the data values of the adjacent patches, in order
  to minimize cross context data dependency of execution of expressions for
  each PatchEngine. In general it is an error to specify a internal guard
  layer that is larger than the patch dimension. This is especially important
  to remember when dealing with SparseTileLayout and GridLayout.
! <p>External guard layers are on the edge of the entire data object. Used
  for external boundary conditions. In this diagram, the external guard layers
  are indicated by 'egl'. External guard layers data allocation exists only
  for those PatchEngines that are on the edges of the full domain of the
  Pooma data object. The data elements that make up the external guard layers
! are generaly used for external boundary conditions.
  <p>In this example, the GuardLayers specification is symmetric, and the
  size of the internal and external guard layers are both 2. In general,
  GuardLayers may be specified asymmetrically, for instance:
--- 59,80 ----
  specified by the internal guard layer for those patches that make up the
  edges of the data object.
  <p>There are two types of GuardLayers used by a MultiPatchEngine based
! Pooma data object. These are internal guard layers and external guard
! layers. Each data patch of a POOMA data object has its domain
! extended by the number of data elements specified by the GuardLayers object.
! <p>Internal guard layers are layers of elements or cells added to the upper
  and/or lower end of the sub-domain of the PatchEngine.&nbsp; These cells
  are to be filled with the data values of the adjacent patches, in order
  to minimize cross context data dependency of execution of expressions for
  each PatchEngine. In general it is an error to specify a internal guard
  layer that is larger than the patch dimension. This is especially important
  to remember when dealing with SparseTileLayout and GridLayout.
! <p>External guard layers are on the edge of the entire data object and are used
  for external boundary conditions. In this diagram, the external guard layers
  are indicated by 'egl'. External guard layers data allocation exists only
  for those PatchEngines that are on the edges of the full domain of the
  Pooma data object. The data elements that make up the external guard layers
! are generally used for external boundary conditions.
  <p>In this example, the GuardLayers specification is symmetric, and the
  size of the internal and external guard layers are both 2. In general,
  GuardLayers may be specified asymmetrically, for instance:
*************** regions for a UniformGridLayout with asy
*** 101,107 ****
  <br>Interval&lt;2> dom(Interval&lt;1>(0,14),Interval&lt;1>(0,9));
  <br>UniformGridPartition&lt;2>&nbsp; asypart(Loc&lt;2>(3,2),igl,egl);
  <br>Array&lt;2,double,MultiPatchEngine&lt;UniformTag,CompressibleBrick>
! >&nbsp;&nbsp;&nbsp;&nbsp; Aarray(UniformGridLayout&lt;2>(dom,asypart,ReplicatedTag());</blockquote>
  <img SRC="/users/luchini/bgraph/bordergraphrevision.gif" ALT="2x3 patch asymetric guard cell" NOSAVE height=398 width=550>
  <br>&nbsp;
  <h1>
--- 101,107 ----
  <br>Interval&lt;2> dom(Interval&lt;1>(0,14),Interval&lt;1>(0,9));
  <br>UniformGridPartition&lt;2>&nbsp; asypart(Loc&lt;2>(3,2),igl,egl);
  <br>Array&lt;2,double,MultiPatchEngine&lt;UniformTag,CompressibleBrick>
! >&nbsp;&nbsp;&nbsp;&nbsp; Array(UniformGridLayout&lt;2>(dom,asypart,ReplicatedTag());</blockquote>
  <img SRC="/users/luchini/bgraph/bordergraphrevision.gif" ALT="2x3 patch asymetric guard cell" NOSAVE height=398 width=550>
  <br>&nbsp;
  <h1>
*************** or overlap onto the data space of patche
*** 129,151 ****
  <br>&nbsp;
  <p><img SRC="grid.2stl.gif" >
  <p><b>DynamicLayout</b>
! <br>An inherently 1 dimensional Layout, that allows the patches to be resized.
  <p><img SRC="grid.2D.gif" >
  <br>&nbsp;
  <p><b>DomainLayout&lt;Dim></b>
  <br>A single patch domain defined by a single Interval.
  <br>&nbsp;
! <h2>
! <a NAME="Ltags></a>UniformGridLayout, GridLayout and SparseTileLayout</b></h2>The UniformGridLayout, GridLayout, and SparseTileLayout all have a trailingtag argument on most of their constructors that specifies how the data is ContextMapper<it>ed</it>. These tags are ReplicatedTag and DistributedTag.If ReplicatedTag is specified, then LocalMapper is used, while if DistributedTagis specified, the the DistributedMapper is used. All of the aforementioned layouts have the default constructor that doesn't require the use of the ReplicatedTag or DistributedTag, and the constructor of the form (Domain,Partitioner, ContextMapper), since it explicitly specifies a ContextMapper,doesn't require the use of the trailing tags.<br>?<h2><a NAME=" Partitioners"></a><b>Partitioners:</b></h2>
  
  <p><br>Layouts have Partitioners that are invoked to generate the
! <br>patch-subdomains, taking into account internal and external GuardLayers.
  <p>The currently available Partitioners are
  <p><b>UniformGridPartitioner&lt;Dim></b>
! <br>Used to generate patches for a UniformGirdLayout.
  <br>&nbsp;
! <p><b>GirdPartitioner&lt;Dim></b>
! <br>Used to generate patches for a GridLayout
  <p><b>TilePartition&lt;Dim></b>
  <br>Generates patches from a provided list of domains.
  <p><b>SpatialPartition&lt;ReferenceLayout></b>
--- 129,152 ----
  <br>&nbsp;
  <p><img SRC="grid.2stl.gif" >
  <p><b>DynamicLayout</b>
! <br>An inherently 1-dimensional Layout, that allows the patches to be resized.
  <p><img SRC="grid.2D.gif" >
  <br>&nbsp;
  <p><b>DomainLayout&lt;Dim></b>
  <br>A single patch domain defined by a single Interval.
  <br>&nbsp;
! <h2><a NAME="Ltags"></a>UniformGridLayout, GridLayout and SparseTileLayout</b></h2>The UniformGridLayout, GridLayout, and SparseTileLayout all have a trailing tag argument on most of their constructors that specifies how the data is ContextMapper<it>ed</it>. These tags are ReplicatedTag and DistributedTag. If ReplicatedTag is specified, then LocalMapper is used, while if DistributedTag is specified, then the DistributedMapper is used. All of the aforementioned layouts have the default constructor that doesn't require the use of the ReplicatedTag or DistributedTag.  A constructor having the form (Domain,Partitioner, ContextMapper) doesn't require the use of the trailing tags since it explicitly specifies a ContextMapper.<br>
! 
! <h2><a NAME=" Partitioners"></a><b>Partitioners:</b></h2>
  
  <p><br>Layouts have Partitioners that are invoked to generate the
! patch-subdomains, taking into account internal and external GuardLayers.
  <p>The currently available Partitioners are
  <p><b>UniformGridPartitioner&lt;Dim></b>
! <br>Used to generate patches for a UniformGridLayout.
  <br>&nbsp;
! <p><b>GridPartitioner&lt;Dim></b>
! <br>Used to generate patches for a GridLayout.
  <p><b>TilePartition&lt;Dim></b>
  <br>Generates patches from a provided list of domains.
  <p><b>SpatialPartition&lt;ReferenceLayout></b>
*************** contexts with an approximately minimum s
*** 181,189 ****
  <br>&nbsp;
  <br>&nbsp;
  <p><b>ContiguousMapper</b>
! <br>Assigns patches to a context in a modified Fortran storage order: as
  the index gets to an boundary, the lowest axis index decrements, rather
! than going from LowIndex = IndexMax to LowIndex = 0; See the figure for
  an illustration of this mapper.
  <p><img SRC="/am/mother/u0/luchini/cmapper/grid.2cmapper.gif" ALT="contiguous mapper" NOSAVE >
  <br>&nbsp;
--- 182,190 ----
  <br>&nbsp;
  <br>&nbsp;
  <p><b>ContiguousMapper</b>
! <br>Assigns patches to a context in a modified Fortran storage order: As
  the index gets to an boundary, the lowest axis index decrements, rather
! than going from LowIndex = IndexMax to LowIndex = 0; see the figure for
  an illustration of this mapper.
  <p><img SRC="/am/mother/u0/luchini/cmapper/grid.2cmapper.gif" ALT="contiguous mapper" NOSAVE >
  <br>&nbsp;
*************** patch engines must not be distributed.
*** 207,212 ****
  <p>Similarly, MultiPatchEngine may not be constructed with the PatchEngine
  tag specified as Remote&lt;PatchEngine> if LocalMapper is specified in
  the Layout used to construct the MultiPatchEngine. Either of the aforementioned
! combinations will generate a Pinsist error inside MultiPatchEngine.
  </body>
  </html>
--- 208,213 ----
  <p>Similarly, MultiPatchEngine may not be constructed with the PatchEngine
  tag specified as Remote&lt;PatchEngine> if LocalMapper is specified in
  the Layout used to construct the MultiPatchEngine. Either of the aforementioned
! combinations will generate a PInsist error inside MultiPatchEngine.
  </body>
  </html>
Index: src/Engine/IndexFunctionEngine.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Engine/IndexFunctionEngine.h,v
retrieving revision 1.23
diff -c -p -r1.23 IndexFunctionEngine.h
*** src/Engine/IndexFunctionEngine.h	2001/08/30 01:15:04	1.23
--- src/Engine/IndexFunctionEngine.h	2001/11/30 04:22:46
*************** struct IndexFunctionView
*** 87,93 ****
  // 
  // Typedefs for the tag, element types, domain and dimensions.
  // Operator() with integers to evaluate elements quickly.
! // Operator() with a doman to subset.
  // Accessor for the domain.
  //-----------------------------------------------------------------------------
  
--- 87,93 ----
  // 
  // Typedefs for the tag, element types, domain and dimensions.
  // Operator() with integers to evaluate elements quickly.
! // Operator() with a domain to subset.
  // Accessor for the domain.
  //-----------------------------------------------------------------------------
  
Index: src/Engine/NotifyEngineWrite.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Engine/NotifyEngineWrite.h,v
retrieving revision 1.6
diff -c -p -r1.6 NotifyEngineWrite.h
*** src/Engine/NotifyEngineWrite.h	2000/05/10 05:07:37	1.6
--- src/Engine/NotifyEngineWrite.h	2001/11/30 04:22:46
***************
*** 42,49 ****
  //-----------------------------------------------------------------------------
  // Overview: 
  //
! // NotifyEngineWrite is a general wrapper class the is used to tell an engine
! // that we're going to write to it.
  //-----------------------------------------------------------------------------
  
  //-----------------------------------------------------------------------------
--- 42,49 ----
  //-----------------------------------------------------------------------------
  // Overview: 
  //
! // NotifyEngineWrite is a general wrapper class that is used to tell
! // an engine that we're going to write to it.
  //-----------------------------------------------------------------------------
  
  //-----------------------------------------------------------------------------
Index: src/Engine/UserFunction.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Engine/UserFunction.h,v
retrieving revision 1.28
diff -c -p -r1.28 UserFunction.h
*** src/Engine/UserFunction.h	2001/09/14 22:37:57	1.28
--- src/Engine/UserFunction.h	2001/11/30 04:22:46
*************** template<int D, class T, class E> class 
*** 100,106 ****
  //     inherits from UserFunction below.
  //
  //   Expression: The type of the expression to which the function
! //      is being applied.  This should be a Array.
  //
  //-----------------------------------------------------------------------------
  
--- 100,106 ----
  //     inherits from UserFunction below.
  //
  //   Expression: The type of the expression to which the function
! //      is being applied.  This should be an Array.
  //
  //-----------------------------------------------------------------------------
  
Index: src/Field/Mesh/UniformRectilinearMesh.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Field/Mesh/UniformRectilinearMesh.h,v
retrieving revision 1.2
diff -c -p -r1.2 UniformRectilinearMesh.h
*** src/Field/Mesh/UniformRectilinearMesh.h	2001/09/20 22:07:32	1.2
--- src/Field/Mesh/UniformRectilinearMesh.h	2001/11/30 04:22:46
***************
*** 47,57 ****
  #include "Field/FieldEngine/FieldEnginePatch.h" // Used in ctors
  #include "Field/Mesh/NoMesh.h"                  // Base class
  #include "Field/FieldCentering.h"               // Centering<Dim> inline
! #include "Tiny/Vector.h"                           // Class member
  
  //-----------------------------------------------------------------------------
  // Holds the data for a uniform rectilinear mesh. That class has a ref-counted
! // instance of this class
  //-----------------------------------------------------------------------------
  
  template <int Dim, class T>
--- 47,57 ----
  #include "Field/FieldEngine/FieldEnginePatch.h" // Used in ctors
  #include "Field/Mesh/NoMesh.h"                  // Base class
  #include "Field/FieldCentering.h"               // Centering<Dim> inline
! #include "Tiny/Vector.h"                        // Class member
  
  //-----------------------------------------------------------------------------
  // Holds the data for a uniform rectilinear mesh. That class has a ref-counted
! // instance of this class.
  //-----------------------------------------------------------------------------
  
  template <int Dim, class T>
Index: src/Layout/UniformGridLayout.cpp
===================================================================
RCS file: /home/pooma/Repository/r2/src/Layout/UniformGridLayout.cpp,v
retrieving revision 1.36
diff -c -p -r1.36 UniformGridLayout.cpp
*** src/Layout/UniformGridLayout.cpp	2001/04/18 02:19:09	1.36
--- src/Layout/UniformGridLayout.cpp	2001/11/30 04:22:47
*************** void UniformGridLayoutData<Dim>::calcGCF
*** 435,441 ****
  // Repartition the layout using a new Partitioner scheme.  The initial
  // domain lists are cleared out, the partitioner is invoked, and then
  // all the observers are notified.  This can only be done with a
! // GridParition partitioner.
  //
  //-----------------------------------------------------------------------------
  
--- 435,441 ----
  // Repartition the layout using a new Partitioner scheme.  The initial
  // domain lists are cleared out, the partitioner is invoked, and then
  // all the observers are notified.  This can only be done with a
! // GridPartition partitioner.
  //
  //-----------------------------------------------------------------------------
  
Index: src/Layout/UniformGridLayout.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Layout/UniformGridLayout.h,v
retrieving revision 1.83
diff -c -p -r1.83 UniformGridLayout.h
*** src/Layout/UniformGridLayout.h	2001/06/05 18:42:12	1.83
--- src/Layout/UniformGridLayout.h	2001/11/30 04:22:47
*************** public:
*** 1030,1037 ****
    typedef UniformGridLayout<Dim>               This_t; // for convenience
    typedef Observable<This_t>                   Observable_t;
  
! //   // Iterator through nodes. Basically the same as the vector iterator
! //   // except it dereferences automatically.
    
    typedef DerefIterator<Value_t>               iterator;
    typedef ConstDerefIterator<Value_t>          const_iterator;
--- 1030,1037 ----
    typedef UniformGridLayout<Dim>               This_t; // for convenience
    typedef Observable<This_t>                   Observable_t;
  
!   // Iterator through nodes. Basically the same as the vector iterator
!   // except it dereferences automatically.
    
    typedef DerefIterator<Value_t>               iterator;
    typedef ConstDerefIterator<Value_t>          const_iterator;
Index: src/Partition/DistributedMapper.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Partition/DistributedMapper.h,v
retrieving revision 1.7
diff -c -p -r1.7 DistributedMapper.h
*** src/Partition/DistributedMapper.h	2001/10/15 17:34:31	1.7
--- src/Partition/DistributedMapper.h	2001/11/30 04:22:48
*************** public:
*** 63,69 ****
    {
      int ncontexts = Pooma::contexts();
      int npc = templist.size()/ncontexts;
!     // if there are more contexts then patches, assign one
      // patch per context for as many patches as there are. 
      if(ncontexts> templist.size())
        {
--- 63,69 ----
    {
      int ncontexts = Pooma::contexts();
      int npc = templist.size()/ncontexts;
!     // If there are more contexts than patches, assign one
      // patch per context for as many patches as there are. 
      if(ncontexts> templist.size())
        {
Index: src/Partition/GridPartition.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Partition/GridPartition.h,v
retrieving revision 1.28
diff -c -p -r1.28 GridPartition.h
*** src/Partition/GridPartition.h	2001/06/28 19:08:11	1.28
--- src/Partition/GridPartition.h	2001/11/30 04:22:48
*************** class UniformGridPartition;
*** 88,126 ****
  // sub-domain specifications along each axis, or any of the specifiers used
  // for the UniformGridPartition. 
  //
! // GridPartition inherets from UniformGridPartition
  //
  // A GridPartition object is constructed with the following information:
  //   GridPartition()
  //   Creates one partition, with no guard cells
  //
  //  GridPartition(const Loc<Dim> &n, int p=-1)
! //Creates n[i] blocks along each i'th dimension
  //
  //  GridPartition(const Loc<Dim> &n, 
  //                       const GuardLayers<Dim> &gcs)
! //Same as above, with internal and external guard cell sizes set to gcs.
  // 
  //                       
  // GridPartition(const Loc<Dim> &n, 
  //                       const GuardLayers<Dim> &igcs,
  //                       const GuardLayers<Dim> &egcs)
! //Same as above, with internal and external guard cell sizes specified 
! //independently.
  //
  //  GridPartition(const Grid<Dim> &g)
! //Partitions according to the Grid object.
  //
  //  GridPartition(const Grid<Dim> &g, 
  //                       const GuardLayers<Dim> &gcs)
! //Same as above, with internal and external guard cell sizes set to gcs.
  // 
  //                       
  // GridPartition(const Grid<Dim> &g, 
  //                       const GuardLayers<Dim> &igcs,
  //                       const GuardLayers<Dim> &egcs)
! //Same as above, with internal and external guard cell sizes specified 
! //independently.
  //-------------------------------------------------------------------------
  
  template<int Dim>
--- 88,126 ----
  // sub-domain specifications along each axis, or any of the specifiers used
  // for the UniformGridPartition. 
  //
! // GridPartition inherits from UniformGridPartition.
  //
  // A GridPartition object is constructed with the following information:
  //   GridPartition()
  //   Creates one partition, with no guard cells
  //
  //  GridPartition(const Loc<Dim> &n, int p=-1)
! // Creates n[i] blocks along each i'th dimension
  //
  //  GridPartition(const Loc<Dim> &n, 
  //                       const GuardLayers<Dim> &gcs)
! // Same as above, with internal and external guard cell sizes set to gcs.
  // 
  //                       
  // GridPartition(const Loc<Dim> &n, 
  //                       const GuardLayers<Dim> &igcs,
  //                       const GuardLayers<Dim> &egcs)
! // Same as above, with internal and external guard cell sizes specified 
! // independently.
  //
  //  GridPartition(const Grid<Dim> &g)
! // Partitions according to the Grid object.
  //
  //  GridPartition(const Grid<Dim> &g, 
  //                       const GuardLayers<Dim> &gcs)
! // Same as above, with internal and external guard cell sizes set to gcs.
  // 
  //                       
  // GridPartition(const Grid<Dim> &g, 
  //                       const GuardLayers<Dim> &igcs,
  //                       const GuardLayers<Dim> &egcs)
! // Same as above, with internal and external guard cell sizes specified 
! // independently.
  //-------------------------------------------------------------------------
  
  template<int Dim>
Index: src/Partition/UniformGridPartition.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Partition/UniformGridPartition.h,v
retrieving revision 1.26
diff -c -p -r1.26 UniformGridPartition.h
*** src/Partition/UniformGridPartition.h	2001/10/10 00:16:03	1.26
--- src/Partition/UniformGridPartition.h	2001/11/30 04:22:48
*************** public:
*** 256,468 ****
      return hasGuards_m; 
    }
  
! bool hasInternalGuards() const 
! { 
!   return hasGuards_m && internalGuards_m != 0;
! }
  
! bool hasExternalGuards() const 
! { 
!   return hasGuards_m && externalGuards_m != 0; 
! }
  
! const GuardLayers<Dim> &internalGuards() const 
! { 
!   return internalGuards_m; 
! }
    
! const GuardLayers<Dim> &externalGuards() const 
! { 
!   return externalGuards_m; 
! }
  
  
! //============================================================
! // Partition methods
! //============================================================
  
! // For the given global domain, partition it into subdomains and put
! // the results in the provided layout object by calling
! // 'layoutData.addDomainList(List_t &templist)'.  Return the
! // total number of subdomains added.
    
    
! template<class D>
! int partition(const D &domain, 
! 	      List_t & all,
! 	      const ContextMapper<Dim>& cmapper) const 
! {
    
!   // The type info for domain we should be creating for the layout.
  
!   typedef typename DomainTraits<Domain_t>::Element_t Element_t;
  
!   // Make sure we have the right dimensionality.
      
!   CTAssert(Dim == DomainTraits<D>::dimensions);
!   CTAssert(Dim == DomainTraits<Domain_t>::dimensions);
  
!   // This will only work with UnitStride domains
      
!   CTAssert(DomainTraits<D>::unitStride == 1);
!   CTAssert(DomainTraits<Domain_t>::unitStride == 1);
  
!   // make sure the list is empty
  
!   PAssert(all.size() == 0);
  
!   // Cache the origin of the domain and make sure the domain is
!   // properly sized. Also, build a domain corresponding to the
!   // number of blocks in each direction for iterating over below.
      
!   Element_t origin[Dim];
!   Element_t sizes[Dim];
!   Interval<Dim> bdomain = Pooma::NoInit(); // dummy initializer
         
!   int i;
  
!   for (i = 0; i < Dim; ++i) 
!     {
!       if (!domain.empty())
! 	{
! 	  int gcwidth = 
! 	    (internalGuards_m.lower(i) > internalGuards_m.upper(i)) ?
! 	    internalGuards_m.lower(i) : internalGuards_m.upper(i);
! 
! 	  PInsist((domain[i].length() % blocks()[i].first()) == 0,
! 		  "All the blocks in a grid must be the same size.");
! 
! 	  origin[i]  = domain[i].first();
! 	  sizes[i]   = domain[i].length() / blocks()[i].first();
! 
! 	  PInsist(sizes[i] >= gcwidth,
! 		  "Block sizes too small for guard layer specification.");
! 	}
!       bdomain[i] = Interval<1>(blocks()[i].first());
!     }
! 
!   // Loop over all the blocks, creating new domains. 
! 
!   typename Interval<Dim>::const_iterator it = bdomain.begin();
!   while (it != bdomain.end()) 
!     {
!       // Start with an initially empty domain and empty guard cells.
  	
!       Domain_t owned;
!       GuardLayers<Dim> iguards(0);
!       GuardLayers<Dim> eguards(0);
! 
!       // Calculate the subdomain, if the global domain is not empty.
!       // If it is, we just sue the empty domain.
! 
!       if (!domain.empty())
! 	{
! 	  Loc<Dim> pos = *it;
! 	  for (i = 0; i < Dim; ++i) 
! 	    {
! 	      int position = pos[i].first();
! 	      Element_t a = origin[i] + sizes[i]*position;
! 	      Element_t b = a + sizes[i] - 1;
! 	      typedef typename 
! 		DomainTraits<Domain_t>::OneDomain_t OneDomain_t;
! 	      owned[i] = OneDomain_t(a, b);
! 	    }
  
! 	  // Calculate the internal and external guard layer specifications
! 	  // for this domain.
          
! 	  if (hasGuards_m)
! 	    {
! 	      iguards = internalGuards_m;
              
! 	      // Check if we're at an edge, and if so use the
! 	      // external specfication for that edge.
                  
! 	      for (int d = 0; d < Dim; ++d)
! 		{
! 		  int position = pos[d].first();
! 		  if ( position == bdomain[d].first() ) 
! 		    {
! 		      eguards.lower(d) = externalGuards_m.lower(d);
! 		      iguards.lower(d) = 0;
! 		    }
! 		  if ( position == bdomain[d].last() ) 
! 		    {
! 		      eguards.upper(d) = externalGuards_m.upper(d);
! 		      iguards.upper(d) = 0;
! 		    }
! 		}
! 	    }
! 	}
!       typename Value_t::ID_t gid = all.size();
!       typename Value_t::ID_t lid = (-1);
  
!       // Create a new Node object to store the subdomain data.
  	
!       GuardLayers<Dim>::addGuardLayers(owned,eguards);
  
!       Domain_t allocated = owned;
  
!       GuardLayers<Dim>::addGuardLayers(allocated,iguards);
  
!       Value_t *node = new Value_t(owned, allocated, -1, gid, lid);
  
!       all.push_back(node);
  
!       // Increment our counters and iterators.
        
!       ++it;
!     }
  
!   cmapper.map(all);
  
!   // At the end, return # of domains created.
      
!   return num_m;
! }
  
! template<class D>
! int partition(const D &domain, List_t & list) const 
! {
!   return partition(domain,list,DefaultMapper_t(*this));
! }
  
  protected:
  
! // The number of blocks along each dimension.
    
! Loc<Dim> blocks_m;
    
! // Do we have guard layers?
    
! bool hasGuards_m;
    
! // Are the external guards different from the internal?
    
! bool hasCustomEdgeGuards_m;
    
! // Specification of internal guard layers.
    
! GuardLayers<Dim> internalGuards_m;
    
! // Specification of external guard layers.
    
! GuardLayers<Dim> externalGuards_m;
  
! // The total number of blocks to create.
    
! int num_m;
    
! // Calculate num_m from blocks_m:
    
! void calcNum()
! {
!   num_m = blocks_m[0].first();
!   for (int d = 1; d < Dim; ++d) 
!     {
!       num_m *= blocks_m[d].first();
!     }
! }
      
  };
  
--- 256,468 ----
      return hasGuards_m; 
    }
  
!   bool hasInternalGuards() const 
!   { 
!     return hasGuards_m && internalGuards_m != 0;
!   }
  
!   bool hasExternalGuards() const 
!   { 
!     return hasGuards_m && externalGuards_m != 0; 
!   }
  
!   const GuardLayers<Dim> &internalGuards() const 
!   { 
!     return internalGuards_m; 
!   }
    
!   const GuardLayers<Dim> &externalGuards() const 
!   { 
!     return externalGuards_m; 
!   }
  
  
!   //============================================================
!   // Partition methods
!   //============================================================
  
!   // For the given global domain, partition it into subdomains and put
!   // the results in the provided layout object by calling
!   // 'layoutData.addDomainList(List_t &templist)'.  Return the
!   // total number of subdomains added.
    
    
!   template<class D>
!   int partition(const D &domain, 
! 		List_t & all,
! 		const ContextMapper<Dim>& cmapper) const 
!   {
    
!     // The type info for domain we should be creating for the layout.
  
!     typedef typename DomainTraits<Domain_t>::Element_t Element_t;
  
!     // Make sure we have the right dimensionality.
      
!     CTAssert(Dim == DomainTraits<D>::dimensions);
!     CTAssert(Dim == DomainTraits<Domain_t>::dimensions);
  
!     // This will only work with UnitStride domains
      
!     CTAssert(DomainTraits<D>::unitStride == 1);
!     CTAssert(DomainTraits<Domain_t>::unitStride == 1);
  
!     // make sure the list is empty
  
!     PAssert(all.size() == 0);
  
!     // Cache the origin of the domain and make sure the domain is
!     // properly sized. Also, build a domain corresponding to the
!     // number of blocks in each direction for iterating over below.
      
!     Element_t origin[Dim];
!     Element_t sizes[Dim];
!     Interval<Dim> bdomain = Pooma::NoInit(); // dummy initializer
         
!     int i;
  
!     for (i = 0; i < Dim; ++i) 
!       {
! 	if (!domain.empty())
! 	  {
! 	    int gcwidth = 
! 	      (internalGuards_m.lower(i) > internalGuards_m.upper(i)) ?
! 	      internalGuards_m.lower(i) : internalGuards_m.upper(i);
! 
! 	    PInsist((domain[i].length() % blocks()[i].first()) == 0,
! 		    "All the blocks in a grid must be the same size.");
! 
! 	    origin[i]  = domain[i].first();
! 	    sizes[i]   = domain[i].length() / blocks()[i].first();
! 
! 	    PInsist(sizes[i] >= gcwidth,
! 		    "Block sizes too small for guard layer specification.");
! 	  }
! 	bdomain[i] = Interval<1>(blocks()[i].first());
!       }
! 
!     // Loop over all the blocks, creating new domains. 
! 
!     typename Interval<Dim>::const_iterator it = bdomain.begin();
!     while (it != bdomain.end()) 
!       {
! 	// Start with an initially empty domain and empty guard cells.
  	
! 	Domain_t owned;
! 	GuardLayers<Dim> iguards(0);
! 	GuardLayers<Dim> eguards(0);
! 
! 	// Calculate the subdomain, if the global domain is not empty.
! 	// If it is, we just use the empty domain.
! 
! 	if (!domain.empty())
! 	  {
! 	    Loc<Dim> pos = *it;
! 	    for (i = 0; i < Dim; ++i) 
! 	      {
! 		int position = pos[i].first();
! 		Element_t a = origin[i] + sizes[i]*position;
! 		Element_t b = a + sizes[i] - 1;
! 		typedef typename 
! 		  DomainTraits<Domain_t>::OneDomain_t OneDomain_t;
! 		owned[i] = OneDomain_t(a, b);
! 	      }
  
! 	    // Calculate the internal and external guard layer specifications
! 	    // for this domain.
          
! 	    if (hasGuards_m)
! 	      {
! 		iguards = internalGuards_m;
              
! 		// Check if we're at an edge, and if so use the
! 		// external specfication for that edge.
                  
! 		for (int d = 0; d < Dim; ++d)
! 		  {
! 		    int position = pos[d].first();
! 		    if ( position == bdomain[d].first() ) 
! 		      {
! 			eguards.lower(d) = externalGuards_m.lower(d);
! 			iguards.lower(d) = 0;
! 		      }
! 		    if ( position == bdomain[d].last() ) 
! 		      {
! 			eguards.upper(d) = externalGuards_m.upper(d);
! 			iguards.upper(d) = 0;
! 		      }
! 		  }
! 	      }
! 	  }
! 	typename Value_t::ID_t gid = all.size();
! 	typename Value_t::ID_t lid = (-1);
  
! 	// Create a new Node object to store the subdomain data.
  	
! 	GuardLayers<Dim>::addGuardLayers(owned,eguards);
  
! 	Domain_t allocated = owned;
  
! 	GuardLayers<Dim>::addGuardLayers(allocated,iguards);
  
! 	Value_t *node = new Value_t(owned, allocated, -1, gid, lid);
  
! 	all.push_back(node);
  
! 	// Increment our counters and iterators.
        
! 	++it;
!       }
  
!     cmapper.map(all);
  
!     // At the end, return # of domains created.
      
!     return num_m;
!   }
  
!   template<class D>
!   int partition(const D &domain, List_t & list) const 
!   {
!     return partition(domain,list,DefaultMapper_t(*this));
!   }
  
  protected:
  
!   // The number of blocks along each dimension.
    
!   Loc<Dim> blocks_m;
    
!   // Do we have guard layers?
    
!   bool hasGuards_m;
    
!   // Are the external guards different from the internal?
    
!   bool hasCustomEdgeGuards_m;
    
!   // Specification of internal guard layers.
    
!   GuardLayers<Dim> internalGuards_m;
    
!   // Specification of external guard layers.
    
!   GuardLayers<Dim> externalGuards_m;
  
!   // The total number of blocks to create.
    
!   int num_m;
    
!   // Calculate num_m from blocks_m:
    
!   void calcNum()
!   {
!     num_m = blocks_m[0].first();
!     for (int d = 1; d < Dim; ++d) 
!       {
! 	num_m *= blocks_m[d].first();
!       }
!   }
      
  };
  
Index: src/Tulip/CollectFromContexts.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Tulip/CollectFromContexts.h,v
retrieving revision 1.1
diff -c -p -r1.1 CollectFromContexts.h
*** src/Tulip/CollectFromContexts.h	2001/09/13 20:40:54	1.1
--- src/Tulip/CollectFromContexts.h	2001/11/30 04:22:48
*************** public:
*** 126,132 ****
    static inline int size(const CollectionValue<T> &v)
    {
      int nBytes = Serialize<CHEETAH, bool>::size(v.valid());
!     nBytes += Serialize<CHEETAH, int>::size(v.context())
      if (v.valid())
        nBytes += Serialize<CHEETAH, T>::size(v.value());
  
--- 126,132 ----
    static inline int size(const CollectionValue<T> &v)
    {
      int nBytes = Serialize<CHEETAH, bool>::size(v.valid());
!     nBytes += Serialize<CHEETAH, int>::size(v.context());
      if (v.valid())
        nBytes += Serialize<CHEETAH, T>::size(v.value());
  
Index: src/Utilities/Benchmark.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Utilities/Benchmark.h,v
retrieving revision 1.28
diff -c -p -r1.28 Benchmark.h
*** src/Utilities/Benchmark.h	2001/07/25 16:04:12	1.28
--- src/Utilities/Benchmark.h	2001/11/30 04:22:48
***************
*** 43,49 ****
  
  
  //-----------------------------------------------------------------------------
! // Implementaion provides a framework for implementing a benchmark in a
  // specific way. It is a virtual base class. Users must override almost
  // all of the member functions.
  //-----------------------------------------------------------------------------
--- 43,49 ----
  
  
  //-----------------------------------------------------------------------------
! // Implementation provides a framework for implementing a benchmark in a
  // specific way. It is a virtual base class. Users must override almost
  // all of the member functions.
  //-----------------------------------------------------------------------------
Index: src/Utilities/Conform.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Utilities/Conform.h,v
retrieving revision 1.2
diff -c -p -r1.2 Conform.h
*** src/Utilities/Conform.h	2000/03/07 13:18:23	1.2
--- src/Utilities/Conform.h	2001/11/30 04:22:48
***************
*** 28,34 ****
  //-----------------------------------------------------------------------------
  // Overview:
  // A tag for checking whether the terms in an expression have
! // conformant domains.
  //-----------------------------------------------------------------------------
  
  #ifndef POOMA_UTILITIES_CONFORM_H
--- 28,34 ----
  //-----------------------------------------------------------------------------
  // Overview:
  // A tag for checking whether the terms in an expression have
! // conforming domains.
  //-----------------------------------------------------------------------------
  
  #ifndef POOMA_UTILITIES_CONFORM_H
Index: src/Utilities/ModelElement.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Utilities/ModelElement.h,v
retrieving revision 1.2
diff -c -p -r1.2 ModelElement.h
*** src/Utilities/ModelElement.h	2000/03/07 13:18:25	1.2
--- src/Utilities/ModelElement.h	2001/11/30 04:22:48
***************
*** 37,43 ****
  //-----------------------------------------------------------------------------
  // Overview: 
  // 
! // ConstField : A read-only version of Field.
  //-----------------------------------------------------------------------------
  
  template<class T>
--- 37,44 ----
  //-----------------------------------------------------------------------------
  // Overview: 
  // 
! // ModelElement<T>
! //   A wrapper class used to differentiate overloaded functions.
  //-----------------------------------------------------------------------------
  
  template<class T>
Index: src/Utilities/Statistics.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Utilities/Statistics.h,v
retrieving revision 1.1
diff -c -p -r1.1 Statistics.h
*** src/Utilities/Statistics.h	2000/04/12 00:30:06	1.1
--- src/Utilities/Statistics.h	2001/11/30 04:22:48
***************
*** 31,38 ****
  
  //-----------------------------------------------------------------------------
  // Classes: 
! //   Implementation
! //   Benchmark
  //-----------------------------------------------------------------------------
  
  #include <string>
--- 31,38 ----
  
  //-----------------------------------------------------------------------------
  // Classes: 
! //   Statistics
! //   StatisticsData: helper class
  //-----------------------------------------------------------------------------
  
  #include <string>
Index: src/Utilities/WrappedInt.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Utilities/WrappedInt.h,v
retrieving revision 1.10
diff -c -p -r1.10 WrappedInt.h
*** src/Utilities/WrappedInt.h	2000/05/23 23:18:44	1.10
--- src/Utilities/WrappedInt.h	2001/11/30 04:22:48
***************
*** 40,46 ****
  // Helper class: WrappedInt<int> 
  //
  // A tag class templated on an integer.  This class is intended to be
! // used to let you specialize a funtion on a compile time number.
  //
  // For example, if you have an object of type T which you want to pass
  // to a subroutine foo, but you want to specialize that subroutine based on
--- 40,46 ----
  // Helper class: WrappedInt<int> 
  //
  // A tag class templated on an integer.  This class is intended to be
! // used to let you specialize a function on a compile time number.
  //
  // For example, if you have an object of type T which you want to pass
  // to a subroutine foo, but you want to specialize that subroutine based on

From oldham at codesourcery.com  Tue Dec  4 01:08:27 2001
From: oldham at codesourcery.com (Jeffrey Oldham)
Date: Mon, 3 Dec 2001 17:08:27 -0800
Subject: Patch: Preliminary Pooma Manual
Message-ID: <20011203170827.A17307@codesourcery.com>

I have been working on a manual for Pooma2.4.  These files will be
part of the manual, mostly written using DocBook.  A portion of the
tutorial introduction chapter and some instructions regarding
downloading and compiling have been written.  The rest is an outline
also containing notes to myself.  Although these files are a work in
progress, I wish to add these files to the Pooma source tree so others
can look at them and so they will be backed up outside of Mountain
View.

2001-Dec-03  Jeffrey D. Oldham  <oldham at codesourcery.com>

        * docs/manual/collateindex.pl: New file copied from DocBook
        distribution.
        * docs/manual/makefile: New file used to create PostScript and
        HTML versions of the manual.
        * docs/manual/outline.xml: New file containing the manual, written
        in DocBook.
        * docs/manual/figures/distributed.mp: New file containing MetaPost
        illustration showing the distributed computing concepts.
        * docs/manual/figures/doof2d.mp: New file containing MetaPost
        illustrations of the Doof2d tutorial implementations.
        * docs/manual/figures/makefile: New file used to create EPS
        figures from the MetaPost files.
        * docs/manual/programs/Doof2d-Array-distributed-annotated.patch:
        New file containing a patch adding callout annotations to
        Doof2d-Array-distributed.cpp.
        * docs/manual/programs/Doof2d-Array-element-annotated.patch:
        Analogous.
        * docs/manual/programs/Doof2d-Array-parallel-annotated.patch:
        Analogous.
        * docs/manual/programs/Doof2d-Array-stencil-annotated.patch:
        Analogous.
        * docs/manual/programs/Doof2d-C-element-annotated.patch:
        Analogous.
        * docs/manual/programs/makefile: New file containing instructions
        to create DocBook versions of programs.
        * examples/Manual/Doof2d/Doof2d-Array-distributed.cpp: New source
        code file containing a simple Doof2d implementation using Arrays
        and distributed computation.
        * examples/Manual/Doof2d/Doof2d-Array-element.cpp: Analogous.
        * examples/Manual/Doof2d/Doof2d-Array-parallel.cpp: Analogous.
        * examples/Manual/Doof2d/Doof2d-Array-stencil.cpp: Analogous.
        * examples/Manual/Doof2d/Doof2d-C-element.cpp: Analogous.
        * examples/Manual/Doof2d/Doof2d-Field-distributed.cpp: Analogous.
        * examples/Manual/Doof2d/Doof2d-Field-parallel.cpp: Analogous.
        * examples/Manual/Doof2d/include.mk: New file needed for
        compilation.
        * examples/Manual/Doof2d/makefile: New file containing instructions
        to compile simple Doof2d implementations.

Thanks,
Jeffrey D. Oldham
oldham at codesourcery.com
-------------- next part --------------
Index: docs/manual/collateindex.pl
===================================================================
RCS file: collateindex.pl
diff -N collateindex.pl
*** /dev/null	Fri Mar 23 21:37:44 2001
--- collateindex.pl	Mon Dec  3 14:01:51 2001
***************
*** 0 ****
--- 1,596 ----
+ # -*- Perl -*-
+ #
+ # $Id: collateindex.pl,v 1.12 2000/01/27 15:07:15 nwalsh Exp $
+ 
+ use Getopt::Std;
+ 
+ $usage = "Usage: $0 <opts> file
+ Where <opts> are:
+        -p        Link to points in the document.  The default is to link
+                  to the closest containing section.
+        -g        Group terms with IndexDiv based on the first letter 
+                  of the term (or its sortas attribute).
+                  (This probably doesn't handle i10n particularly well)
+        -s name   Name the IndexDiv that contains symbols.  The default
+                  is 'Symbols'.  Meaningless if -g is not used.
+        -t name   Title for the index.
+        -P file   Read a preamble from file.  The content of file will
+                  be inserted before the <index> tag.
+        -i id     The ID for the <index> tag.
+        -o file   Output to file. Defaults to stdout.
+        -S scope  Scope of the index, must be 'all', 'local', or 'global'.
+                  If unspecified, 'all' is assumed.
+        -I scope  The implied scope, must be 'all', 'local', or 'global'.
+                  IndexTerms which do not specify a scope will have the
+                  implied scope.  If unspecified, 'all' is assumed.
+        -x        Make a SetIndex.
+        -f        Force the output file to be written, even if it appears
+                  to have been edited by hand.
+        -N        New index (generates an empty index file).
+        file      The file containing index data generated by Jade
+                  with the DocBook HTML Stylesheet.\n";
+ 
+ die $usage if ! getopts('Dfgi:NpP:s:o:S:I:t:x');
+ 
+ $linkpoints   = $opt_p;
+ $lettergroups = $opt_g;
+ $symbolsname  = $opt_s || "Symbols";
+ $title        = $opt_t;
+ $preamble     = $opt_P;
+ $outfile      = $opt_o || '-';
+ $indexid      = $opt_i;
+ $scope        = uc($opt_S) || 'ALL';
+ $impliedscope = uc($opt_I) || 'ALL';
+ $setindex     = $opt_x;
+ $forceoutput  = $opt_f;
+ $newindex     = $opt_N;
+ $debug        = $opt_D;
+ 
+ $indextag     = $setindex ? 'setindex' : 'index';
+ 
+ if ($newindex) {
+     safe_open(*OUT, $outfile);
+     if ($indexid) {
+ 	print OUT "<$indextag id='$indexid'>\n\n";
+     } else {
+ 	print OUT "<$indextag>\n\n";
+     }
+ 
+     print OUT "<!-- This file was produced by collateindex.pl.         -->\n";
+     print OUT "<!-- Remove this comment if you edit this file by hand! -->\n";
+ 
+     print OUT "</$indextag>\n";
+     exit 0;
+ }
+ 
+ $dat = shift @ARGV || die $usage;
+ die "$0: cannot find $dat.\n" if ! -f $dat;
+ 
+ %legal_scopes = ('ALL' => 1, 'LOCAL' => 1, 'GLOBAL' => 1);
+ if ($scope && !$legal_scopes{$scope}) {
+     die "Invalid scope.\n$usage\n";
+ }
+ if ($impliedscope && !$legal_scopes{$impliedscope}) {
+     die "Invalid implied scope.\n$usage\n";
+ }
+ 
+ @term = ();
+ %id   = ();
+ 
+ $termcount = 0;
+ 
+ print STDERR "Processing $dat...\n";
+ 
+ # Read the index file, creating an array of objects.  Each object 
+ # represents and indexterm and has fields for the content of the
+ # indexterm
+ 
+ open (F, $dat);
+ while (<F>) {
+     chop;
+ 
+     if (/^\/indexterm/i) {
+ 	push (@term, $idx);
+ 	next;
+     }
+ 
+     if (/^indexterm (.*)$/i) {
+ 	$termcount++;
+ 	$idx = {};
+ 	$idx->{'zone'} = {};
+ 	$idx->{'href'} = $1;
+ 	$idx->{'count'} = $termcount;
+ 	$idx->{'scope'} = $impliedscope;
+ 	next;
+     }
+ 
+     if (/^indexpoint (.*)$/i) {
+ 	$idx->{'hrefpoint'} = $1;
+ 	next;
+     }
+ 
+     if (/^title (.*)$/i) {
+ 	$idx->{'title'} = $1;
+ 	next;
+     }
+ 
+     if (/^primary[\[ ](.*)$/i) {
+ 	if (/^primary\[(.*?)\] (.*)$/i) {
+ 	    $idx->{'psortas'} = $1;
+ 	    $idx->{'primary'} = $2;
+ 	} else {
+ 	    $idx->{'psortas'} = $1;
+ 	    $idx->{'primary'} = $1;
+ 	}
+ 	next;
+     }
+ 
+     if (/^secondary[\[ ](.*)$/i) {
+ 	if (/^secondary\[(.*?)\] (.*)$/i) {
+ 	    $idx->{'ssortas'} = $1;
+ 	    $idx->{'secondary'} = $2;
+ 	} else {
+ 	    $idx->{'ssortas'} = $1;
+ 	    $idx->{'secondary'} = $1;
+ 	}
+ 	next;
+     }
+ 
+     if (/^tertiary[\[ ](.*)$/i) {
+ 	if (/^tertiary\[(.*?)\] (.*)$/i) {
+ 	    $idx->{'tsortas'} = $1;
+ 	    $idx->{'tertiary'} = $2;
+ 	} else {
+ 	    $idx->{'tsortas'} = $1;
+ 	    $idx->{'tertiary'} = $1;
+ 	}
+ 	next;
+     }
+ 
+     if (/^see (.*)$/i) {
+ 	$idx->{'see'} = $1;
+ 	next;
+     }
+ 
+     if (/^seealso (.*)$/i) {
+ 	$idx->{'seealso'} = $1;
+ 	next;
+     }
+ 
+     if (/^significance (.*)$/i) {
+ 	$idx->{'significance'} = $1;
+ 	next;
+     }
+ 
+     if (/^class (.*)$/i) {
+ 	$idx->{'class'} = $1;
+ 	next;
+     }
+ 
+     if (/^scope (.*)$/i) {
+ 	$idx->{'scope'} = uc($1);
+ 	next;
+     }
+ 
+     if (/^startref (.*)$/i) {
+ 	$idx->{'startref'} = $1;
+ 	next;
+     }
+ 
+     if (/^id (.*)$/i) {
+ 	$idx->{'id'} = $1;
+ 	$id{$1} = $idx;
+ 	next;
+     }
+ 
+     if (/^zone (.*)$/i) {
+ 	my($href) = $1;
+ 	$_ = scalar(<F>);
+ 	chop;
+ 	die "Bad zone: $_\n" if !/^title (.*)$/i;
+ 	$idx->{'zone'}->{$href} = $1;
+ 	next;
+     }
+ 
+     die "Unrecognized: $_\n";
+ }
+ close (F);
+ 
+ print STDERR "$termcount entries loaded...\n";
+ 
+ # Fixup the startrefs...
+ # In DocBook, STARTREF is a #CONREF attribute; support this by copying
+ # all of the fields from the indexterm with the id specified by STARTREF
+ # to the indexterm that has the STARTREF.
+ foreach $idx (@term) {
+     my($ididx, $field);
+     if ($idx->{'startref'}) {
+ 	$ididx = $id{$idx->{'startref'}};
+ 	foreach $field ('primary', 'secondary', 'tertiary', 'see', 'seealso',
+ 		        'psortas', 'ssortas', 'tsortas', 'significance',
+ 		        'class', 'scope') {
+ 	    $idx->{$field} = $ididx->{$field};
+ 	}
+     }
+ }
+ 
+ # Sort the index terms
+ @term = sort termsort @term;
+ 
+ # Move all of the non-alphabetic entries to the front of the index.
+ @term = sortsymbols(@term);
+ 
+ safe_open(*OUT, $outfile);
+ 
+ # Write the index...
+ if ($indexid) {
+     print OUT "<$indextag id='$indexid'>\n\n";
+ } else {
+     print OUT "<$indextag>\n\n";
+ }
+ 
+ print OUT "<!-- This file was produced by collateindex.pl.         -->\n";
+ print OUT "<!-- Remove this comment if you edit this file by hand! -->\n";
+ 
+ print OUT "<!-- ULINK is abused here.
+       
+       The URL attribute holds the URL that points from the index entry
+       back to the appropriate place in the output produced by the HTML
+       stylesheet. (It's much easier to calculate this URL in the first
+       pass.)
+ 
+       The Role attribute holds the ID (either real or manufactured) of
+       the corresponding INDEXTERM.  This is used by the print backends
+       to produce page numbers.
+ 
+       The entries below are sorted and collated into the correct order.
+       Duplicates may be removed in the HTML backend, but in the print
+       backends, it is impossible to suppress duplicate pages or coalesce
+       sequences of pages into a range.
+ -->\n\n";
+ 
+ print OUT "<title>$title</title>\n\n" if $title;
+ 
+ $last = {};     # the last indexterm we processed
+ $first = 1;     # this is the first one
+ $group = "";    # we're not in a group yet
+ $lastout = "";  # we've not put anything out yet
+ 
+ foreach $idx (@term) {
+     next if $idx->{'startref'}; # no way to represent spans...
+     next if ($idx->{'scope'} eq 'LOCAL') && ($scope eq 'GLOBAL');
+     next if ($idx->{'scope'} eq 'GLOBAL') && ($scope eq 'LOCAL');
+     next if &same($idx, $last); # suppress duplicates
+ 
+     $termcount--;
+ 
+     # If primary changes, output a whole new index term, otherwise just
+     # output another secondary or tertiary, as appropriate.  We know from
+     # sorting that the terms will always be in the right order.
+     if (!&tsame($last, $idx, 'primary')) { 
+ 	print "DIFF PRIM\n" if $debug;
+ 	&end_entry() if not $first;
+ 
+ 	if ($lettergroups) {
+ 	    # If we're grouping, make the right indexdivs
+ 	    $letter = $idx->{'psortas'};
+ 	    $letter = $idx->{'primary'} if !$letter;
+ 	    $letter = uc(substr($letter, 0, 1));
+ 	    
+ 	    # symbols are a special case
+ 	    if (($letter lt 'A') || ($letter gt 'Z')) {
+ 		if (($group eq '')
+ 		    || (($group ge 'A') && ($group le 'Z'))) {
+ 		    print OUT "</indexdiv>\n" if !$first;
+ 		    print OUT "<indexdiv><title>$symbolsname</title>\n\n";
+ 		    $group = $letter;
+ 		}
+ 	    } elsif (($group eq '') || ($group ne $letter)) {
+ 		print OUT "</indexdiv>\n" if !$first;
+ 		print OUT "<indexdiv><title>$letter</title>\n\n";
+ 		$group = $letter;
+ 	    }
+ 	}
+ 
+ 	$first = 0; # there can only be on first ;-)
+ 
+ 	print OUT "<indexentry>\n";
+ 	print OUT "  <primaryie>", $idx->{'primary'};
+ 	$lastout = "primaryie";
+ 
+  	if ($idx->{'secondary'}) {
+ 	    print OUT "\n  </primaryie>\n";
+ 	    print OUT "  <secondaryie>", $idx->{'secondary'};
+ 	    $lastout = "secondaryie";
+ 	};
+ 
+ 	if ($idx->{'tertiary'}) {
+ 	    print OUT "\n  </secondaryie>\n";
+ 	    print OUT "  <tertiaryie>", $idx->{'tertiary'};
+ 	    $lastout = "tertiaryie";
+ 	}
+     } elsif (!&tsame($last, $idx, 'secondary')) {
+ 	print "DIFF SEC\n" if $debug;
+ 
+ 	print OUT "\n  </$lastout>\n" if $lastout;
+ 
+ 	print OUT "  <secondaryie>", $idx->{'secondary'};
+ 	$lastout = "secondaryie";
+ 	if ($idx->{'tertiary'}) {
+ 	    print OUT "\n  </secondaryie>\n";
+ 	    print OUT "  <tertiaryie>", $idx->{'tertiary'};
+ 	    $lastout = "tertiaryie";
+ 	}
+     } elsif (!&tsame($last, $idx, 'tertiary')) {
+ 	print "DIFF TERT\n" if $debug;
+ 
+ 	print OUT "\n  </$lastout>\n" if $lastout;
+ 
+ 	if ($idx->{'tertiary'}) {
+ 	    print OUT "  <tertiaryie>", $idx->{'tertiary'};
+ 	    $lastout = "tertiaryie";
+ 	}
+     }
+ 
+     &print_term($idx);
+     
+     $last = $idx;
+ }
+ 
+ # Termcount is > 0 iff some entries were skipped.
+ print STDERR "$termcount entries ignored...\n";
+ 
+ &end_entry();
+ 
+ print OUT "</indexdiv>\n" if $lettergroups;
+ print OUT "</$indextag>\n";
+ 
+ close (OUT);
+ 
+ print STDERR "Done.\n";
+ 
+ sub same {
+     my($a) = shift;
+     my($b) = shift;
+ 
+     my($aP) = $a->{'psortas'} || $a->{'primary'};   
+     my($aS) = $a->{'ssortas'} || $a->{'secondary'}; 
+     my($aT) = $a->{'tsortas'} || $a->{'tertiary'};  
+ 	                                            
+     my($bP) = $b->{'psortas'} || $b->{'primary'};   
+     my($bS) = $b->{'ssortas'} || $b->{'secondary'}; 
+     my($bT) = $b->{'tsortas'} || $b->{'tertiary'};  
+ 
+     my($same);
+ 
+     $aP =~ s/^\s*//; $aP =~ s/\s*$//; $aP = uc($aP);
+     $aS =~ s/^\s*//; $aS =~ s/\s*$//; $aS = uc($aS);
+     $aT =~ s/^\s*//; $aT =~ s/\s*$//; $aT = uc($aT);
+     $bP =~ s/^\s*//; $bP =~ s/\s*$//; $bP = uc($bP);
+     $bS =~ s/^\s*//; $bS =~ s/\s*$//; $bS = uc($bS);
+     $bT =~ s/^\s*//; $bT =~ s/\s*$//; $bT = uc($bT);
+ 
+ #    print "[$aP]=[$bP]\n";
+ #    print "[$aS]=[$bS]\n";
+ #    print "[$aT]=[$bT]\n";
+ 
+     # Two index terms are the same if:
+     # 1. the primary, secondary, and tertiary entries are the same
+     #    (or have the same SORTAS)
+     # AND
+     # 2. They occur in the same titled section
+     # AND
+     # 3. They point to the same place
+     #
+     # Notes: Scope is used to suppress some entries, but can't be used
+     #          for comparing duplicates.
+     #        Interpretation of "the same place" depends on whether or
+     #          not $linkpoints is true.
+ 
+     $same = (($aP eq $bP)
+ 	     && ($aS eq $bS)
+ 	     && ($aT eq $bT)
+ 	     && ($a->{'title'} eq $b->{'title'})
+ 	     && ($a->{'href'} eq $b->{'href'}));
+ 
+     # If we're linking to points, they're only the same if they link
+     # to exactly the same spot.  (surely this is redundant?)
+     $same = $same && ($a->{'hrefpoint'} eq $b->{'hrefpoint'})
+ 	if $linkpoints;
+ 
+     $same;
+ }
+ 
+ sub tsame {
+     # Unlike same(), tsame only compares a single term
+     my($a) = shift;
+     my($b) = shift;
+     my($term) = shift;
+     my($sterm) = substr($term, 0, 1) . "sortas";
+     my($A, $B);
+ 
+     $A = $a->{$sterm} || $a->{$term};
+     $B = $b->{$sterm} || $b->{$term};
+ 
+     $A =~ s/^\s*//; $A =~ s/\s*$//; $A = uc($A);
+     $B =~ s/^\s*//; $B =~ s/\s*$//; $B = uc($B);
+ 
+     return $A eq $B;
+ }
+ 
+ sub end_entry {
+     # End any open elements...
+     print OUT "\n  </$lastout>\n" if $lastout;
+     print OUT "</indexentry>\n\n";
+     $lastout = "";
+ }
+ 
+ sub print_term {
+     # Print out the links for an indexterm.  There can be more than
+     # one if the term has a ZONE that points to more than one place.
+     # (do we do the right thing in that case?)
+     my($idx) = shift;
+     my($key, $indent, @hrefs);
+     my(%href) = ();
+     my(%phref) = ();
+ 
+     $indent = "    ";
+ 
+     if ($idx->{'see'}) {
+ 	# it'd be nice to make this a link...
+ 	if ($lastout) {
+ 	    print OUT "\n  </$lastout>\n";
+ 	    $lastout = "";
+ 	}
+ 	print OUT $indent, "<seeie>", $idx->{'see'}, "</seeie>\n";
+ 	return;
+     }
+ 
+     if ($idx->{'seealso'}) {
+ 	# it'd be nice to make this a link...
+ 	if ($lastout) {
+ 	    print OUT "\n  </$lastout>\n";
+ 	    $lastout = "";
+ 	}
+ 	print OUT $indent, "<seealsoie>", $idx->{'seealso'}, "</seealsoie>\n";
+ 	return;
+     }
+ 
+     if (keys %{$idx->{'zone'}}) {
+ 	foreach $key (keys %{$idx->{'zone'}}) {
+ 	    $href{$key} = $idx->{'zone'}->{$key};
+ 	    $phref{$key} = $idx->{'zone'}->{$key};
+ 	}
+     } else {
+ 	$href{$idx->{'href'}} = $idx->{'title'};
+ 	$phref{$idx->{'href'}} = $idx->{'hrefpoint'};
+     }
+ 
+     # We can't use <LINK> because we don't know the ID of the term in the
+     # original source (and, in fact, it might not have one).
+     print OUT ",\n";
+     @hrefs = keys %href;
+     while (@hrefs) {
+ 	my($linkend) = "";
+ 	my($role) = "";
+ 	$key = shift @hrefs;
+ 	if ($linkpoints) {
+ 	    $linkend = $phref{$key};
+ 	} else {
+ 	    $linkend = $key;
+ 	}
+ 
+ 	$role = $linkend;
+ 	$role = $1 if $role =~ /\#(.*)$/;
+ 
+ 	print OUT $indent;
+ 	print OUT "<ulink url=\"$linkend\" role=\"$role\">";
+ 	print OUT "<emphasis>" if ($idx->{'significance'} eq 'PREFERRED');
+ 	print OUT $href{$key};
+ 	print OUT "</emphasis>" if ($idx->{'significance'} eq 'PREFERRED');
+ 	print OUT "</ulink>";
+     }
+ }
+ 
+ sub termsort {
+     my($aP) = $a->{'psortas'} || $a->{'primary'};   
+     my($aS) = $a->{'ssortas'} || $a->{'secondary'}; 
+     my($aT) = $a->{'tsortas'} || $a->{'tertiary'};  
+     my($ap) = $a->{'count'};
+ 	                                            
+     my($bP) = $b->{'psortas'} || $b->{'primary'};   
+     my($bS) = $b->{'ssortas'} || $b->{'secondary'}; 
+     my($bT) = $b->{'tsortas'} || $b->{'tertiary'};  
+     my($bp) = $b->{'count'};
+ 
+     $aP =~ s/^\s*//; $aP =~ s/\s*$//; $aP = uc($aP);
+     $aS =~ s/^\s*//; $aS =~ s/\s*$//; $aS = uc($aS);
+     $aT =~ s/^\s*//; $aT =~ s/\s*$//; $aT = uc($aT);
+     $bP =~ s/^\s*//; $bP =~ s/\s*$//; $bP = uc($bP);
+     $bS =~ s/^\s*//; $bS =~ s/\s*$//; $bS = uc($bS);
+     $bT =~ s/^\s*//; $bT =~ s/\s*$//; $bT = uc($bT);
+ 
+     if ($aP eq $bP) {
+ 	if ($aS eq $bS) {
+ 	    if ($aT eq $bT) {
+ 		# make sure seealso's always sort to the bottom
+ 		return 1 if ($a->{'seealso'});
+ 		return -1  if ($b->{'seealso'});
+ 		# if everything else is the same, keep these elements
+ 		# in document order (so the index links are in the right
+ 		# order)
+ 		return $ap <=> $bp;
+ 	    } else {
+ 		return $aT cmp $bT;
+ 	    }
+ 	} else {
+ 	    return $aS cmp $bS;
+ 	}
+     } else {
+ 	return $aP cmp $bP;
+     }
+ }
+ 
+ sub sortsymbols {
+     my(@term) = @_;
+     my(@new) = ();
+     my(@sym) = ();
+     my($letter);
+     my($idx);
+ 
+     # Move the non-letter things to the front.  Should digits be thier
+     # own group?  Maybe...
+     foreach $idx (@term) {
+ 	$letter = $idx->{'psortas'};
+ 	$letter = $idx->{'primary'} if !$letter;
+ 	$letter = uc(substr($letter, 0, 1));
+ 
+ 	if (($letter lt 'A') || ($letter gt 'Z')) {
+ 	    push (@sym, $idx);
+ 	} else {
+ 	    push (@new, $idx);
+ 	}
+     }
+ 
+     return (@sym, @new);
+ }
+ 
+ sub safe_open {
+     local(*OUT) = shift;
+     local(*F, $_);
+ 
+     if (($outfile ne '-') && (!$forceoutput)) {
+ 	my($handedit) = 1;
+ 	if (open (OUT, $outfile)) {
+ 	    while (<OUT>) {
+ 		if (/<!-- Remove this comment if you edit this file by hand! -->/){
+ 		    $handedit = 0;
+ 		    last;
+ 		}
+ 	    } 
+ 	    close (OUT);
+ 	} else {
+ 	    $handedit = 0;
+ 	}
+ 	
+ 	if ($handedit) {
+ 	    print "\n$outfile appears to have been edited by hand; use -f or\n";
+ 	    print "      change the output file.\n";
+ 	    exit 1;
+ 	}
+     }
+ 
+     open (OUT, ">$outfile") || die "$usage\nCannot write to $outfile.\n";
+ 
+     if ($preamble) { 
+ 	# Copy the preamble
+ 	if (open(F, $preamble)) {
+ 	    while (<F>) {
+ 		print OUT $_;
+ 	    }
+ 	    close(F);
+ 	} else {
+ 	    warn "$0: cannot open preamble $preamble.\n";
+ 	}
+     }
+ }
Index: docs/manual/makefile
===================================================================
RCS file: makefile
diff -N makefile
*** /dev/null	Fri Mar 23 21:37:44 2001
--- makefile	Mon Dec  3 14:01:51 2001
***************
*** 0 ****
--- 1,69 ----
+ ### Oldham, Jeffrey D.
+ ### 1997 Dec 26
+ ### misc
+ ###
+ ### LaTeX -> PostScript/PDF/WWW
+ ### XML -> TeX/DVI/PS/PDF
+ 
+ # Definitions for PostScript and WWW Creation
+ TEX=		latex
+ WWWHOMEDIR=	/u/oldham/www
+ LATEX2HTML=	latex2html
+ BASICLATEX2HTMLOPTIONS=	-info "" -no_footnode -no_math -html_version 3.2,math
+ #LATEX2HTMLOPTIONS=	-local_icons -split +1 $(BASICLATEX2HTMLOPTIONS)
+ LATEX2HTMLOPTIONS=	-no_navigation -split 0 $(BASICLATEX2HTMLOPTIONS)
+ MPOST=		mpost
+ 
+ # Definitions for Jade.
+ JADEDIR=		/usr/lib/sgml/stylesheets/docbook
+ PRINTDOCBOOKDSL=	print/docbook.dsl
+ HTMLDOCBOOKDSL=		html/docbook.dsl
+ XML=			dtds/decls/xml.dcl
+ INDEXOPTIONS=		-t 'Index' -i 'index' -g -p
+ 
+ CXXFLAGS= -g -Wall -pedantic -W -Wstrict-prototypes -Wpointer-arith -Wbad-function-cast -Wcast-align -Wconversion -Wnested-externs -Wundef -Winline -static
+ 
+ all: outline.ps
+ 
+ %.all:	%.ps %.pdf %.html
+ 	chmod 644 $*.ps $*.pdf
+ 	mv $*.ps $*.pdf $*
+ 
+ %.dvi:	%.ltx
+ 	$(TEX) $<
+ 	# bibtex $*
+ 	# $(TEX) $<
+ 	$(TEX) $<
+ 
+ %.ps:	%.dvi
+ 	dvips -t letter $< -o
+ 
+ %.pdf.ltx:	%.ltx
+ 	sed -e 's/^%\\usepackage{times}/\\usepackage{times}/' $< > $@
+ 
+ %.pdf:	%.pdf.ps
+ 	ps2pdf $< $@
+ 
+ # This rule assumes index creation.
+ %.dvi:	%.xml genindex.sgm
+ 	jade -D$(JADEDIR) -t sgml -d $(HTMLDOCBOOKDSL) -V html-index $(XML) $<
+ 	perl collateindex.pl $(INDEXOPTIONS) -o genindex.sgm HTML.index
+ 	jade -D$(JADEDIR) -t tex -d $(PRINTDOCBOOKDSL) $(XML) $< && jadetex $*.tex && jadetex $*.tex && jadetex $*.tex
+ 
+ genindex.sgm:
+ 	perl collateindex.pl $(INDEXOPTIONS) -N -o $@
+ 
+ %.html:	%.xml
+ 	jade -D$(JADEDIR) -t sgml -d $(HTMLDOCBOOKDSL) $(XML) $<
+ 
+ %.pdf:	%.xml
+ 	jade -D$(JADEDIR) -t tex -d $(PRINTDOCBOOKDSL) $(XML) $< && pdfjadetex $*.tex && pdfjadetex $*.tex
+ 
+ mproof-%.ps:	%.mp
+ 	declare -x TEX=latex && $(MPOST) $< && tex mproof.tex $*.[0-9]* && dvips mproof.dvi -o $@
+ 
+ %.txt:	%.ltx
+ 	detex $< > $@
+ 
+ clean:
+ 	rm -f *.dvi *.aux *.log *.toc *.bak *.blg *.bbl *.glo *.idx *.lof *.lot *.htm *.mpx mpxerr.tex HTML.index outline.tex
Index: docs/manual/outline.xml
===================================================================
RCS file: outline.xml
diff -N outline.xml
*** /dev/null	Fri Mar 23 21:37:44 2001
--- outline.xml	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,4287 ----
+ <?xml version="1.0"?>
+ 
+ <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN" "file://usr/lib/sgml/docbookx.dtd" [
+ 
+ <!-- Text to Check, Revise, Replace -->
+   <!-- FIXME: Revise before releasing. -->
+   <!-- UPDATE: Check before publishing to see if any needs changing. -->
+   <!-- ADD: Write more material. -->
+ 
+ <!-- Index Entity Declarations -->
+ <!ENTITY genindex.sgm SYSTEM "genindex.sgm">
+ 
+ <!-- General Entity Declarations -->
+ 
+ <!ENTITY book "book">
+   <!-- Produce a notation for the book/manual/report/WWW page.  -->
+   <!-- Modify this to the desired noun. -->
+ <!ENTITY Book "Book">
+   <!-- Produce a capitalized version of &book;  -->
+   <!-- Modify this to the desired noun. -->
+ <!ENTITY c "<application class='software'>C</application>">
+   <!-- Produce a notation for the C programming language.  -->
+   <!-- Modify this to the desired formatting. -->
+ <!ENTITY cc "<application class='software'>C++</application>">
+   <!-- Produce a notation for the C++ programming language.  -->
+   <!-- Modify this to the desired formatting. -->
+ <!ENTITY cheetah "<application class='software'>Cheetah</application>" >
+   <!-- Produce a notation for the Cheetah Library.  -->
+ <!ENTITY doof2d "<command>Doof2d</command>" >
+   <!-- Produce a notation for the Doof2d program.  -->
+ <!ENTITY make "<application class='software'>Make</application>">
+   <!-- Produce a notation for the GNU Make program.  -->
+ <!ENTITY mm "<application class='software'>MM</application>">
+   <!-- Produce a notation for the MM Library.  -->
+ <!ENTITY mpi "<application class='software'>MPI</application>">
+   <!-- Produce a notation for the MPI package.  -->
+ <!ENTITY pdt "<application class='software'>PDToolkit</application>">
+   <!-- Produce a notation for the PDT software package.  -->
+ <!ENTITY pete "<application class='software'>PETE</application>">
+   <!-- Produce a notation for the PETE library.  -->
+ <!ENTITY pooma "<application class='software'>POOMA</application>">
+   <!-- Produce a notation for Pooma software.  -->
+ <!ENTITY poomaToolkit "<application class='software'>POOMA Toolkit</application>">
+   <!-- Produce a notation for the Pooma toolkit.  -->
+ <!ENTITY purify "<application class='software'>Purify</application>">
+   <!-- Produce a notation for the Purify library.  -->
+ <!ENTITY smarts "<application class='software'>Smarts</application>">
+   <!-- Produce a notation for the Smarts software package.  -->
+   <!-- Modify this after capitalization is decided. -->
+ <!ENTITY stl "<application class='software'>STL</application>">
+   <!-- Produce a notation for the C++ Standard Template Library software package.  -->
+ <!ENTITY tau "<application class='software'>Tau</application>">
+   <!-- Produce a notation for the Tau software package.  -->
+ 
+ <!-- Type Entity Declarations -->
+ 
+ <!ENTITY array "<type>Array</type>">
+   <!-- The "Array" type. -->
+ <!ENTITY benchmark "<type>Benchmark</type>">
+   <!-- The "Benchmark" type. -->
+ <!ENTITY brick "<type>Brick</type>">
+   <!-- The "Brick" engine type. -->
+ <!ENTITY compressiblebrick "<type>CompressibleBrick</type>">
+   <!-- The "CompressibleBrick" engine type. -->
+ <!ENTITY distributedtag "<type>DistributedTag</type>">
+   <!-- The DistributedTag Layout type. -->
+ <!ENTITY domain "<type>Domain</type>">
+   <!-- The "Domain" type. -->
+ <!ENTITY double "<type>double</type>">
+   <!-- The C "double" type. -->
+ <!ENTITY dynamicarray "<type>DynamicArray</type>">
+   <!-- The "DynamicArray" type. -->
+ <!ENTITY engine "<type>Engine</type>">
+   <!-- The "Engine" type. -->
+ <!ENTITY field "<type>Field</type>">
+   <!-- The "Field" type. -->
+ <!ENTITY interval "<type>Interval</type>">
+   <!-- The "Interval" type. -->
+ <!ENTITY layout "<type>Layout</type>">
+   <!-- The "Layout" type without template parameters. -->
+ <!ENTITY leaffunctor "<type>LeafFunctor</type>">
+   <!-- The "LeafFunctor" type. -->
+ <!ENTITY multipatch "<type>MultiPatch</type>">
+   <!-- The "MultiPatch" engine without template parameters. -->
+ <!ENTITY replicatedtag "<type>ReplicatedTag</type>">
+   <!-- The ReplicatedTag Layout type. -->
+ <!ENTITY stencil "<type>Stencil</type>">
+   <!-- The "Stencil" type. -->
+ <!ENTITY vector "<type>Vector</type>">
+   <!-- The "Vector" type. -->
+ 
+ <!-- Mathematical Entity Declarations -->
+ 
+ <!ENTITY n "n">
+   <!-- the size of one dimension of an array -->
+ 
+ <!-- System and Operating System Entity Declarations -->
+ <!ENTITY gcc "<application>g++</application>">
+   <!-- The GNU Compiler Collection C++ compiler. -->
+ <!ENTITY kcc "<application>KCC</application>">
+   <!-- The KAI C++ compiler. -->
+ <!ENTITY linux "<application>Linux</application>">
+   <!-- The Linux operating system. -->
+ 
+ <!-- &pooma; URLs and Files -->
+ 
+ <!ENTITY poomaDownloadPage '<ulink url="http://pooma.codesourcery.com/pooma/download">http://pooma.codesourcery.com/pooma/download</ulink>'>
+   <!-- The WWW page supporting downloading the &pooma; source code. -->
+   <!-- UPDATE this URL. -->
+ <!ENTITY poomaHomePage '<ulink url="http://www.pooma.com/">http://www.pooma.com/</ulink>'>
+   <!-- The canonical Pooma home page. -->
+   <!-- UPDATE this filename. -->
+ <!ENTITY poomaSource "pooma-2.3.0">
+   <!-- The Pooma source code directory. -->
+ <!ENTITY poomaSourceFile "&poomaSource;.tgz">
+   <!-- The Pooma source code archive. -->
+ <!ENTITY poomaExampleDirectory "examples/Manual">
+   <!-- The directory holding this manual's example codes. -->
+ 
+ <!-- Spelling and Formatting Decisions -->
+ <!ENTITY author "author">
+   <!-- A word describing an author xor authors. -->
+   <!-- spelling: element-wise, not elementwise -->
+   <!-- phrase: function object, not functor -->
+   <!-- spelling: multidimensional, not multi-dimensional -->
+   <!-- spelling: multiprocessor, not multi-processor -->
+   <!-- spelling: nonzero, not non-zero -->
+ 
+ <!-- External File Entities -->
+ <!ENTITY doof2d-c-element SYSTEM "./programs/Doof2d-C-element-annotated.cpp">
+   <!-- hand-coded Doof2d implementation -->
+ <!ENTITY doof2d-array-element SYSTEM "./programs/Doof2d-Array-element-annotated.cpp">
+   <!-- Array element-wise Doof2d implementation -->
+ <!ENTITY doof2d-array-parallel SYSTEM "./programs/Doof2d-Array-parallel-annotated.cpp">
+   <!-- Array data-parallel Doof2d implementation -->
+ <!ENTITY doof2d-array-stencil SYSTEM "./programs/Doof2d-Array-stencil-annotated.cpp">
+   <!-- Array stencil Doof2d implementation -->
+ <!ENTITY doof2d-array-distributed SYSTEM "./programs/Doof2d-Array-distributed-annotated.cpp">
+   <!-- distributed Array stencil Doof2d implementation -->
+ ]>
+ 
+ <book>
+  <bookinfo>
+   <title>&pooma;</title>
+   <subtitle>A &cc; Toolkit for High-Performance Parallel Scientific Computing</subtitle>
+   <author><firstname>Jeffrey</firstname><othername
+   role='mi'>D.</othername><surname>Oldham</surname>
+    <affiliation>
+     <orgname>CodeSourcery, LLC</orgname>
+    </affiliation>
+   </author>
+   <copyright><year>2001</year><holder>CodeSourcery, LLC (<ulink url="http://www.codesourcery.com"></ulink>)</holder></copyright>
+   <contractsponsor>Los Alamos National Laboratory<ulink url="http://www.lanl.gov"></ulink></contractsponsor>
+   <legalnotice>
+    <!-- FIXME: What is the correct legal notice? -->
+    <para>All rights reserved.  This document may not be redistributed in any form without the express permission of the author.</para>
+   </legalnotice>
+   <revhistory>
+    <revision>
+     <revnumber>0.01</revnumber>
+     <date>2001 Nov 26</date>
+     <authorinitials>jdo</authorinitials>
+     <revremark>first draft</revremark>
+    </revision>
+   </revhistory>
+  </bookinfo>
+ 
+  <!-- FINISH: May we have a short table of contents followed by a -->
+  <!-- complete table of contents? -->
+ 
+  <preface id="preface">
+   <title>Preface</title>
+ 
+   <para>FINISH: Describe the target audience for &pooma; programs and
+   for this manual: &cc; programmers writing scientific code, possibly
+   parallel execution.</para>
+ 
+   <para>Assume familiarity with &cc; template programming and the
+   standard template library.  FIXME: Remove this index
+   entry.<indexterm id="oldham"><primary>Oldham,
+   Jeffrey&nbsp;D.</primary></indexterm></para>
+ 
+   <section id="preface-notation">
+    <title>Notation</title>
+ 
+    <para>UNFINISHED</para>
+   </section>
+ 
+ 
+   <section id="preface-reading_book:">
+    <title>How to Read This &Book;</title>
+ 
+    <para>FINISH: Write this section in a style similar to Lamport's
+    LaTeX section 1.2.  FINISH: Fix the book title and the section
+    number.</para>
+   </section>
+ 
+ 
+   <section id="preface-downloading">
+    <title>Obtaining &pooma; and Sample Programs</title>
+ 
+    <para>Available for free from what WWW site?  Include what portions
+    of <filename class="libraryfile">LICENSE</filename>?  Be sure to
+    include CVS instructions as well.</para>
+ 
+    <para>Which additional packages are necessary and when?</para>
+ 
+   </section>
+ 
+ 
+   <section id="preface-using_modifying">
+    <title>Using and Modifying &pooma;</title>
+ 
+    <para>&pooma; is available under open source license.  It can be
+    used and modified by anyone, anywhere.  Can it be sold?  Include
+    <filename class="libraryfile">LICENSE</filename>.</para>
+ 
+    <para>QUESTION: How do developers contribute code?</para>
+ 
+   </section>
+ 
+  </preface>
+ 
+ 
+  <part id="programming">
+   <title>Programming with &pooma;</title>
+ 
+   <chapter id="introduction">
+    <title>Introduction</title>
+ 
+    <para>QUESTION: Add a partintro to the part above?</para>
+ 
+    <para>&pooma; abbreviates <quote>Parallel Object-Oriented Methods
+    and Application</quote>.</para>
+ 
+    <para>This document is an introduction to &pooma; v2.1, a &cc;
+    toolkit for high-performance scientific computation.  &pooma;
+    runs efficiently on single-processor desktop machines,
+    shared-memory multiprocessors, and parallel supercomputers
+    containing dozens or hundreds of processors. What's more, by making
+    extensive use of the advanced features of the ANSI/ISO &cc;
+    standard&mdash;particularly templates&mdash;&pooma; presents a
+    compact, easy-to-read interface to its users.</para>
+ 
+    <para>From Section&nbsp; of
+    <filename>papers/iscope98.pdf</filename>:</para>
+ 
+    <para>Scientific software developers have struggled with the need
+    to express mathematical abstractions in an elegant and maintainable
+    way without sacrificing performance.  The &pooma; (Parallel
+    Object-Oriented Methods and Applications) framework, written in
+    <acronym>ANSI</acronym>/<acronym>ISO</acronym> &cc;, has
+    demonstrated both high expressiveness and high performance for
+    large-scale scientific applications on platforms ranging from
+    workstations to massively parallel supercomputers.  &pooma; provides
+    high-level abstractions for multidimensional arrays, physical
+    meshes, mathematical fields, and sets of particles.  &pooma; also
+    exploits techniques such as expression templates to optimize serial
+    performance while encapsulating the details of parallel
+    communication and supporting block-based data compression.
+    Consequently, scientists can quickly assemble parallel simulation
+    codes by focusing directly on the physical abstractions relevant to
+    the system under study and not the technical difficulties of
+    parallel communication and machine-specific optimization.</para>
+ 
+    <para>ADD: diagram of science and &pooma;.  See the diagram that
+    Mark and I wrote.</para>
+ 
+ 
+    <section id="introduction-pooma_evolution">
+     <title>Evolution of &pooma;</title>
+ 
+     <para>QUESTION: Is this interesting?  Even if it is, it should be
+     short.</para>
+ 
+     <para>The file <filename>papers/SCPaper-95.html</filename>
+     describes ?&pooma;1? and its abstraction layers.</para>
+ 
+     <para>The "Introduction" of
+     <filename>papers/Siam0098.ps</filename> describes the DoE's
+     funding motivation for &pooma;: Accelerated Strategic Computing
+     Initiative (ASCI) and Science-based Stockpile Stewardship (SBSS),
+     pp. 1&ndash;2.</para>
+ 
+     <para>See list of developers on p.&nbsp;1 of
+     <filename>papers/pooma.ps</filename>.</para>
+ 
+     <para>See list of developers on p.&nbsp;1 of
+     <filename>papers/pooma.ps</filename>.  See history and motivation
+     on p.&nbsp;3 of <filename>papers/pooma.ps</filename>.</para>
+ 
+     <para>Use <filename class="libraryfile">README</filename> for
+     information.</para>
+ 
+     <blockquote>
+      <attribution><filename
+ 			    class="libraryfile">introduction.html</filename></attribution>
+ 
+      <para>&pooma; was designed and implemented by scientists working
+      at the Los Alamos National Laboratory's Advanced Computing
+      Laboratory. Between them, these scientists have written and tuned
+      large applications on almost every commercial and experimental
+      supercomputer built in the last two decades. As the technology
+      used in those machines migrates down into departmental computing
+      servers and desktop multiprocessors, &pooma; is a vehicle for its
+      designers' experience to migrate as well. In particular,
+      &pooma;'s authors understand how to get good performance out of
+      modern architectures, with their many processors and multi-level
+      memory hierarchies, and how to handle the subtly complex problems
+      that arise in real-world applications.</para>
+     </blockquote>
+ 
+    </section>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="tutorial">
+    <title>A Tutorial Introduction</title>
+ 
+    <para>UPDATE: In the following paragraph, fix the cross-reference
+    to the actual section.</para>
+ 
+    <para>&pooma; provides different containers and processor
+    configurations and supports different implementation styles, as
+    described in <xref linkend="introduction"></xref>.  In this
+    chapter, we present several different implementations of the
+    &doof2d; two-dimensional diffusion simulation program:
+     <itemizedlist spacing="compact">
+      <listitem>
+       <para>a C-style implementation omitting any use of &pooma;
+       computing each array element individually,</para>
+      </listitem>
+      <listitem>
+       <para>a &pooma; &array; implementation computing each array
+       element individually,</para>
+      </listitem>
+      <listitem>
+       <para>a &pooma; &array; implementation using data-parallel
+       statements,</para>
+      </listitem>
+      <listitem>
+       <para>a &pooma; &array; implementation using stencils, which
+       support local computations,</para>
+      </listitem>
+      <listitem>
+       <para>a stencil-based &pooma; &array; implementation supporting
+       computation on multiple processors</para>
+      </listitem>
+      <listitem>
+       <para>a &pooma; &field; implementation using data-parallel
+       statements, and</para>
+      </listitem>
+      <listitem>
+       <para>a data-parallel &pooma; &field; implementation for
+       multi-processor execution.</para>
+      </listitem>
+     </itemizedlist>
+    </para>
+    <para>These illustrate the &array;, &field;, &engine;, layout,
+    mesh, and domain data types.  They also illustrate various
+    immediate computation styles (element-wise accesses, data-parallel
+    expressions, and stencil computation) and various processor
+    configurations (one sequential processor and multiple
+    processors).</para>
+ 
+    <figure float="1" id="tutorial-doof2d_averagings">
+     <title>&doof2d; Averagings</title>
+     <mediaobject>
+      <imageobject>
+       <imagedata fileref="figures/doof2d.201" format="EPS" align="center"></imagedata>
+      </imageobject>
+      <textobject>
+       <phrase>The Initial Configuration</phrase>
+      </textobject>
+     </mediaobject>
+     <mediaobject>
+      <imageobject>
+       <imagedata fileref="figures/doof2d.202" format="EPS"></imagedata>
+      </imageobject>
+      <textobject>
+       <phrase>After the First Averaging</phrase>
+      </textobject>
+     </mediaobject>
+     <mediaobject>
+      <imageobject>
+       <imagedata fileref="figures/doof2d.203" format="EPS"></imagedata>
+      </imageobject>
+      <textobject>
+       <phrase>After the Second Averaging</phrase>
+      </textobject>
+     </mediaobject>
+    </figure>
+ 
+    <para>The &doof2d; diffusion program starts with a two-dimensional
+    grid of values.  To model an initial density, all grid values are
+    zero except for one nonzero value in the center.  Each averaging,
+    each grid element, except the outermost ones, updates its value by
+    averaging its value and its eight neighbors.  To avoid overwriting
+    grid values before all their uses occur, we use two arrays, reading
+    the first and writing the second and then reversing their roles
+    within each iteration.</para>
+ 
+    <para>Figure <xref linkend="tutorial-doof2d_averagings"></xref>
+    illustrates the averagings.  Initially, only the center element has
+    nonzero value.  To form the first averaging, each element's new
+    value equals the average of its and its neighbors' previous values.
+    Thus, the initial nonzero value spreads to a three-by-three grid.
+    The averaging continues, spreading to a five-by-five grid of
+    nonzero values.  Values in outermost grid cells are always
+    zero.</para>
+ 
+    <para>Before presenting various implementations of %doof2d;, we
+    explain how to install the &poomaToolkit;.</para>
+ 
+    <para>REMOVE: &doof2d; algorithm and code is illustrated in
+    Section&nbsp;4.1 of
+    <filename>pooma-publications/pooma.ps</filename>.  It includes a
+    figure illustrating parallel communication of data.</para>
+ 
+    <section id="tutorial-installation">
+     <title>Installing &pooma;</title>
+ 
+     <para>ADD: How does one install &pooma; using Windows or Mac?</para>
+ 
+     <para>UPDATE: Make a more recent &pooma; source code file
+     available on &poomaDownloadPage;.  For example,
+     <quote>LINUXgcc.conf</quote> is not available.</para>
+ 
+     <para>In this section, we describe how to obtain, build, and
+     install the &poomaToolkit;.  We focus on installing under the
+     Unix operating system.  Instructions for installing on computers
+     running Microsoft Windows or MacOS, as well as more extensive
+     instructions for Unix, appear in <xref
+     linkend="installation"></xref>.</para>
+ 
+     <para>Obtain the &pooma; source code <filename
+     path="http://www.codesourcery.com/pooma/downloads_folder/">&poomaSourceFile;</filename>
+     from the &pooma; download page (&poomaDownloadPage;) available off
+     the &pooma; home page (&poomaHomePage;).  The <quote>tgz</quote>
+     indicates this is a compressed tar archive file.  To extract the
+     source files, use <command>tar xzvf &poomaSourceFile;</command>.
+     Move into the source code directory <filename
+     class="directory">&poomaSource;</filename> directory; e.g.,
+     <command>cd &poomaSource;</command>.</para>
+ 
+     <para>Configuring the source code prepares the necessary paths for
+     compilation.  First, determine a configuration file in
+     corresponding to your operating system and compiler in the
+     <filename class="directory">config/arch/</filename> directory.
+     For example, <filename
+     class="libraryfile">LINUXgcc.conf</filename> supports compiling
+     under a &linux; operating system with &gcc; and <filename
+     class="libraryfile">SGI64KCC.conf</filename> supports compiling
+     under a 64-bit <application>SGI</application> Unix operating
+     system with &kcc;.  Then, configure the source code:
+     <command>./configure --arch LINUXgcc --opt --suite
+     LINUXgcc-opt</command>.  The architecture argument to the
+     <command>--arch</command> option is the name of the corresponding
+     configuration file, omitting its <filename
+     class="libraryfile">.conf</filename> suffix.  The
+     <command>--opt</command> indicates the &poomaToolkit; will
+     contain optimized source code, which makes the code run more
+     quickly but may impede debugging.  Alternatively, the
+     <command>--debug</command> option supports debugging.  The
+     <glossterm linkend="glossary-suite_name">suite name</glossterm>
+     can be any arbitrary string.  We chose
+     <command>LINUXgcc-opt</command> to remind us of the architecture
+     and optimization choice.  <filename
+     class="libraryfile">configure</filename> creates subdirectories
+     named by the suite name <quote>LINUXgcc-opt</quote> for use when
+     compiling the source files.  Comments at the beginning of
+     <filename
+     class="libraryfile">lib/<replaceable>suiteName</replaceable>/PoomaConfiguration.h</filename>
+     record the configuration arguments.</para>
+ 
+     <para>To compile the source code, set the
+     <envar>POOMASUITE</envar> environment variable to the suite name
+     and then type <command>make</command>.  To set the environment
+     variable for the <application>bash</application> shell use
+     <command>export
+     POOMASUITE=<replaceable>suiteName</replaceable></command>,
+     substituting the suite name's
+     <replaceable>suiteName</replaceable>.  For the
+     <application>csh</application> shell, use <command>setenv
+     POOMASUITE LINUXgcc-opt</command>.  Issuing the
+     <command>make</command> command compiles the &pooma; source code
+     files to create the &pooma; library.  The &pooma; makefiles assume
+     the <trademark>GNU</trademark> &make; so substitute the proper
+     command if necessary.  The &pooma; library can be found in, e.g.,
+     <filename
+     class="libraryfile">lib/LINUXgcc-opt/libpooma-gcc.a</filename>.</para>
+    </section>
+ 
+    <section id="tutorial-hand_coded">
+     <title>Hand-Coded Implementation</title>
+ 
+     <para>Before implementing &doof2d; using the &poomaToolkit;, we
+     present a hand-coded implementation of &doof2d;.  See <xref
+     linkend="tutorial-hand_coded-doof2d"></xref>.  After querying the
+     user for the number of averagings, the arrays' memory is
+     allocated.  Since the arrays' size is not known at compile time,
+     the arrays are accesses via pointers to allocated dynamic memory.
+     This memory is deallocated at the program's end to avoid memory
+     leaks.  The arrays are initialized with initial conditions.  For
+     the <varname>b</varname> array, all values except the central ones
+     have nonzero values.  Only the outermost values of the
+     <varname>a</varname> array need be initialized to zero, but we
+     instead initialize them all using the loop used by
+     <varname>b</varname>.</para>
+ 
+     <para>The simulation's kernel consists of triply nested loops.
+     The outermost loop controls the number of iterations.  The inner
+     nested loops iterate through the arrays' elements, excepting the
+     outermost elements; note the loop indices range from 1 to n-2
+     while the array indices range from 0 to n-1.  Each
+     <varname>a</varname> value is assigned the average of its
+     corresponding value in <varname>b</varname> and the latter's
+     neighbors.  Values in the two-dimensional grids are accessed using
+     two sets of brackets, e.g., <statement>a[i][j]</statement>.  After
+     assigning values to <varname>a</varname>, a second averaging reads
+     values in <varname>a</varname>, writing values in
+     <varname>b</varname>.</para>
+ 
+     <para>After the kernel finishes, the final central value is
+     printed.  If the desired number of averagings is even, the value
+     in <varname>b</varname> is printed; otherwise, the value in
+     <varname>a</varname> is used.  Finally, the dynamically-allocated
+     memory must be freed to avoid memory leaks.</para>
+ 
+     <example id="tutorial-hand_coded-doof2d">
+      <title>Hand-Coded Implementation of &doof2d;</title>
+      &doof2d-c-element;
+      <calloutlist>
+       <callout arearefs="tutorial-hand_coded-doof2d-nuaveragings">
+        <para>The user specifies the desired number of averagings.</para>
+       </callout>
+       <callout arearefs="tutorial-hand_coded-doof2d-array_storage">
+        <para>These variables point to the two-dimensional,
+        dynamically-allocated grids so we use a pointer to a pointer to
+        a &double;.</para>
+       </callout>
+       <callout arearefs="tutorial-hand_coded-doof2d-grid_size">
+        <para>The user enters the desired grid size.  The grid will be
+        a square with <varname>n</varname> by <varname
+        lang="c++">n</varname> grid cells.</para>
+       </callout>
+       <callout arearefs="tutorial-hand_coded-doof2d-allocation">
+        <para>Memory for the arrays is allocated.  By default, the
+        array indices are zero-based.</para>
+       </callout>
+       <callout arearefs="tutorial-hand_coded-doof2d-initialization">
+        <para>Initially, all grid values are zero except for the one
+        nonzero value at the center of the second array.  Array
+        positions are indicated using two brackets, e.g.,
+        <statement>a[i][j]</statement>.  A better implementation might
+        initialize only the outermost values of the
+        <varname>a</varname> array.</para>
+       </callout>
+       <callout arearefs="tutorial-hand_coded-doof2d-constants">
+        <para>These constants indicate the number of iterations, and
+        the average weighting.</para>
+       </callout>
+       <callout arearefs="tutorial-hand_coded-doof2d-first_write">
+        <para>Each <varname>a</varname> value, except an outermost one,
+        is assigned the average of its analogous <varname>b</varname>
+        value and that value's neighbors.  Note the loop indices ensure
+        the outermost values are not changed.  The
+        <varname>weight</varname>'s value ensures the computation is an
+        average.</para>
+       </callout>
+       <callout arearefs="tutorial-hand_coded-doof2d-second_write">
+        <para>The second averaging computes <varname>b</varname>'s
+        values using values stored in <varname>a</varname>.</para>
+       </callout>
+       <callout arearefs="tutorial-hand_coded-doof2d-answer">
+        <para>After the averagings finish, the central value is printed.</para>
+       </callout>
+       <callout arearefs="tutorial-hand_coded-doof2d-deallocation">
+        <para>The dynamically-allocated memory must be deallocated to
+        avoid memory leaks.</para>
+       </callout>
+      </calloutlist>
+     </example>
+ 
+     <para>To compile the executable, change directories to the &pooma;
+     <filename
+     class="directory">&poomaExampleDirectory;/Doof2d</filename>
+     directory.  Ensure the <envar>POOMASUITE</envar> environment
+     variable specifies the desired suite name
+     <replaceable>suiteName</replaceable>, as we did when compiling
+     &pooma; in the previous section <xref
+     linkend="tutorial-installation"></xref>.  Issuing the
+     <command>make Doof2d-C-element</command> command creates the
+     executable
+     <command><replaceable>suiteName</replaceable>/Doof2d-C-element</command>.</para>
+ 
+     <para>When running the executable, specify the desired a
+     nonnegative number of averagings and the nonnegative number of
+     grid cells along any dimension.  The resulting grid has the same
+     number of cells along each dimension.  After the executable
+     finishes, the resulting value of the central element is
+     printed.</para>
+    </section>
+ 
+ 
+    <section id="tutorial-array_elementwise">
+     <title>Element-wise &array; Implementation</title>
+ 
+     <para>The simplest way to use the &poomaToolkit; is to
+     use the &pooma; &array; class instead of &c; arrays.  &array;s
+     automatically handle memory allocation and deallocation, support a
+     wider variety of assignments, and can be used in expressions.
+     <xref linkend="tutorial-array_elementwise-doof2d"></xref>
+     implements &doof2d; using &array;s and element-wise accesses.
+     Since the same algorithm is used as <xref
+     linkend="tutorial-hand_coded-doof2d"></xref>, we will concentrate
+     on the differences.</para>
+ 
+     <example id="tutorial-array_elementwise-doof2d">
+      <title>Element-wise &array; Implementation of &doof2d;</title>
+      &doof2d-array-element;
+      <calloutlist>
+       <callout arearefs="tutorial-array_elementwise-doof2d-header">
+        <para>To use &pooma; &array;s, the <filename
+        class="headerfile">Pooma/Arrays.h</filename> must be included.</para>
+       </callout>
+       <callout arearefs="tutorial-array_elementwise-doof2d-pooma_initialize">
+        <para>The &poomaToolkit; structures must be constructed before
+        their use.</para>
+       </callout>
+       <callout arearefs="tutorial-array_elementwise-doof2d-domain">
+        <para>Before creating an &array;, its domain must be specified.
+        The <varname>N</varname> interval represents the
+        one-dimensional integral set {0, 1, 2, &hellip;, n-1}.  An
+        <type>Interval&lt;2&gt;</type> object represents the entire
+        two-dimensional index domain.</para>
+       </callout>
+       <callout arearefs="tutorial-array_elementwise-doof2d-array_creation">
+        <para>An &array;'s template parameters indicate its dimension,
+        its value type, and how the values will be stored or computed.
+        The &brick; &engine; type indicates values will be directly
+        stored.  It is responsible for allocating and deallocating
+        storage so <statement>new</statement> and
+        <statement>delete</statement> statements are not necessary.
+        The <varname>vertDomain</varname> specifies the array index
+        domain.</para>
+       </callout>
+       <callout arearefs="tutorial-array_elementwise-doof2d-initialization">
+        <para>The first statement initializes all &array; values to the
+        same scalar value.  This is possible because each &array;
+        <quote>knows</quote> its domain.  The second statement
+        illustrates &array; element access.  Indices, separated by
+        commas, are surrounded by parentheses rather than surrounded by
+        square brackets (<statement>[]</statement>).</para>
+       </callout>
+       <callout arearefs="tutorial-array_elementwise-doof2d-first_write">
+        <para>&array; element access uses parentheses, rather than
+        square brackets</para>
+       </callout>
+       <callout arearefs="tutorial-array_elementwise-doof2d-blockAndEvaluate">
+        <para>&pooma; may reorder computation of statements.  Calling
+        <function>Pooma::blockAndEvaluate</function> ensures all
+        computation finishes before accessing a particular array
+        element.</para>
+       </callout>
+       <callout arearefs="tutorial-array_elementwise-doof2d-deallocation">
+        <para>Since &array;s are first-class objects, they
+        automatically deallocate any memory they require, eliminating
+        memory leaks.</para>
+       </callout>
+       <callout arearefs="tutorial-array_elementwise-doof2d-pooma_finish">
+        <para>The &poomaToolkit; structures must be destructed after
+        their use.</para>
+       </callout>
+      </calloutlist>
+     </example>
+ 
+     <para>We describe the use of &array; and the &poomaToolkit; in
+     <xref linkend="tutorial-array_elementwise-doof2d"></xref>.
+     &array;s, declared in the <filename
+     class="headerfile">Pooma/Arrays.h</filename>, are first-class
+     objects.  They <quote>know</quote> their index domain, can be used
+     in expressions, can be assigned scalar and array values, and
+     handle their own memory allocation and deallocation.</para>
+ 
+     <para>The creation of the <varname>a</varname> and
+     <varname>b</varname> &array;s requires an object specifying their
+     index domains.  Since these are two-dimensional arrays, their
+     index domains are also two dimensional.  The two-dimensional
+     <type>Interval&lt;2&gt;</type> object is the Cartesian product of
+     two one-dimensional <type>Interval&lt;1&gt;</type> objects, each
+     specifying the integral set {0, 1, 2, &hellip;, n-1}.</para>
+ 
+     <para>An &array;'s template parameters indicate its dimension, the
+     type of its values, and how the values are stored.  Both
+     <varname>a</varname> and <varname>b</varname> are two-dimension
+     arrays storing &double;s so their <varname>dimension</varname>
+     is&nbsp;2 and its element type is &double;.  An &engine; stores an
+     &array;'s values.  For example, a &brick; &engine; explicitly
+     stores all values.  A &compressiblebrick; &engine; also explicitly
+     stores values if more than value is present, but, if all values
+     are the same, storage for just that value is required.  Since an
+     engine can store its values any way it desires, it might instead
+     compute its values using a function or compute the values stored
+     in separate engines.  In practice, most explicitly specified
+     &engine;s are either &brick; or &compressiblebrick;.</para>
+ 
+     <para>&array;s support both element-wise access and scalar
+     assignment.  Element-wise access uses parentheses, not square
+     brackets.  For example, <statement>b(n/2,n/2)</statement>
+     specifies the central element.  The scalar assignment <statement>b
+     = 0.0</statement> assigns the same 0.0 value to all array
+     elements.  This is possible because the array knows the extent of
+     its domain.</para>
+ 
+     <para>After the kernel finishes, the central value is printed out.
+     Just prior to this &array; access, a call to
+     <statement>Pooma::blockAndEvaluate()</statement> ensures all
+     computation has finished.  &pooma; may reorder computation or
+     distribute them among various processors.  Before reading an
+     individual &array; value, <function>blockAndEvaluate</function>
+     ensures the value has the correct value.  Calling this function is
+     necessary only when accessing individual array elements because
+     &pooma; cannot determine when to call the function itself. For
+     example, before printing an array, &pooma; will call
+     <function>blockAndEvaluate</function> itself.</para>
+ 
+     <para>Any program using the &poomaToolkit; must initialize the
+     toolkit's data structures using
+     <statement>Pooma::initialize(argc,argv)</statement>.  This
+     extracts &pooma;-specific command-line options from the
+     command-line arguments in <varname>argv</varname> and initializes
+     the inter-processor communication and other data structures.  When
+     finished, <statement>Pooma::finalize()</statement> ensures all
+     computation has finished and the communication and other data
+     structures are destructed.</para>
+    </section>
+ 
+ 
+    <section id="tutorial-data_parallel">
+     <title>Data-Parallel &array; Implementation</title>
+ 
+     <para>&pooma; supports data-parallel &array; accesses.  Many
+     algorithms are more easily expressed using data-parallel
+     expressions.  Also, the &poomaToolkit; might be able to reorder
+     the data-parallel computations to be more efficient or distribute
+     them among various processors.  In this section, we concentrate
+     the differences between the data-parallel implementation of
+     &doof2d; listed in <xref
+     linkend="tutorial-array_parallel-doof2d"></xref> and the
+     element-wise implementation listed in the previous section <xref
+     linkend="tutorial-array_elementwise"></xref>.</para>
+ 
+     <example id="tutorial-array_parallel-doof2d">
+      <title>Data-Parallel &array; Implementation of &doof2d;</title>
+      &doof2d-array-parallel;
+      <calloutlist>
+       <callout arearefs="tutorial-array_parallel-doof2d-innerdomain">
+        <para>These variables specify one-dimensional domains {1, 2,
+        &hellip;, n-2}.  Their Cartesian product specifies the domain
+        of the array values that are modified.</para>
+       </callout>
+       <callout arearefs="tutorial-array_parallel-doof2d-first_write">
+        <para>Data-parallel expressions replace nested loops and array
+        element accesses.  For example, <statement>a(I,J)</statement>
+        represents the subset of the <varname>a</varname> array having
+        a domain equal to the Cartesian product of <varname>I</varname>
+        and <varname>J</varname>.  Intervals can shifted by an additive
+        or multiplicative constant.</para>
+       </callout>
+      </calloutlist>
+     </example>
+ 
+     <para>Data-parallel expressions apply domain objects to containers
+     to indicate a set of parallel expressions.  For example, in the
+     program listed above, <statement>a(I,J)</statement> specifies all
+     of <varname>a</varname> array excepting the outermost elements.
+     The array's <varname>vertDomain</varname> domain consists of the
+     Cartesian product of {0, 1, 2, &hellip;, n-1} and itself, while
+     <varname>I</varname> and <varname>J</varname> each specify {1, 2,
+     &hellip;, n-2}.  Thus, <statement>a(I,J)</statement> is the subset
+     with a domain of the Cartesian product of {1, 2, &hellip;, n-2}
+     and itself.  It is called a <firstterm>view</firstterm> of an
+     array.  It is itself an array, with a domain and supporting
+     element access, but its storage is the same as
+     <varname>a</varname>'s.  Changing a value in
+     <statement>a(I,J)</statement> also changes the same value in
+     <varname>a</varname>.  Changing a value in the latter also changes
+     the former if the value is not one of <varname>a</varname>'s
+     outermost elements.  The expression
+     <statement>b(I+1,J+1)</statement> indicates the subset of
+     <varname>b</varname> with a domain consisting of the Cartesian
+     product of {2, 3, &hellip;, n-1}, i.e., the same domain as
+     <statement>a(I,J)</statement> but shifted up one unit and to the
+     right one unit.  Only an &interval;'s value, not its name, is
+     important.  Thus, all uses of <varname>J</varname> in this program
+     could be replaced by <varname>I</varname> without changing the
+     semantics.</para>
+ 
+     <figure float="1" id="tutorial-array_parallel-doof2d-adding_arrays">
+      <title>Adding &array;s</title>
+      <mediaobject>
+       <imageobject>
+        <imagedata fileref="figures/doof2d.210" format="EPS" align="center"></imagedata>
+       </imageobject>
+       <textobject>
+        <phrase>Adding two arrays with different domains.</phrase>
+       </textobject>
+       <caption>
+        <para>When adding arrays, values in corresponding positions are
+        added even if they have different indices, indicated by the
+        small numbers adjacent to the arrays.</para>
+       </caption>
+      </mediaobject>
+     </figure>
+ 
+     <para>The statement assigning to <statement>a(I,J)</statement>
+     illustrates that &array;s may participate in expressions.  Each
+     addend is a view of an array, which is itself an array.  Each view
+     has the same domain size so their sum can be formed by
+     corresponding elements of each array.  For example, the lower,
+     left element of the result equals the sum of the lower, left
+     elements of the addend arrays.  For the computation, indices are
+     ignored; only the relative positions within each domain are used.
+     <xref
+     linkend="tutorial-array_parallel-doof2d-adding_arrays"></xref>
+     illustrates adding two arrays with different domain indices.  The
+     indices are indicated by the small numbers to the left and the
+     bottom of the arrays.  Even though 9 and 3 have different indices
+     (1,1) and (2,0), they are added to each other because they have
+     the same relative positions within the addends.</para>
+    </section>
+ 
+ 
+    <section id="tutorial-stencil">
+     <title>Stencil &array; Implementation</title>
+ 
+     <para>Many computations are local, computing a &array;'s value by
+     using close-by &array; values.  Encapsulating this computation in
+     a stencil can yield faster code because the compiler can determine
+     all accesses come from the same array.  Each stencil consists of a
+     function object and an indication of the stencil's extent.</para>
+ 
+     <example id="tutorial-array_stencil-doof2d">
+      <title>Stencil &array; Implementation of &doof2d;</title>
+      &doof2d-array-stencil;
+      <calloutlist>
+       <callout arearefs="tutorial-array_stencil-doof2d-stencil">
+        <para>A stencil is a function object implementing a local
+        operation on an &array;.</para>
+       </callout>
+       <callout
+        arearefs="tutorial-array_stencil-doof2d-stencil_operator">
+        <para>&pooma; applies this function call
+        <function>operator()</function> to the interior domain of an
+        &array;.  Although not strictly necessary, the function's
+        template parameter <varname>C</varname> permits using this
+        stencil with &array;s and other containers.  The
+        <function>read</function> &array; member function supports only
+        reading values, not writing values, thus possibly permitting
+        faster access.</para>
+       </callout>
+       <callout arearefs="tutorial-array_stencil-doof2d-stencil_extent">
+        <para>These two functions indicate the stencil's size.  For
+        each dimension, the stencil extends one cell to the left of (or
+        below) its center and also one call to the right (or above) its
+        center.</para>
+       </callout>
+       <callout
+        arearefs="tutorial-array_stencil-doof2d-stencil_creation">
+        <para>Create the stencil.</para>
+       </callout>
+       <callout arearefs="tutorial-array_stencil-doof2d-first_write">
+        <para>Applying <varname>stencil</varname> to the
+        <varname>b</varname> array and a subset
+        <varname>interiorDomain</varname> of its domain yields an
+        array, which is assigned to a subset of <varname>a</varname>.
+        The stencil's function object is applied to each position in
+        the specified subset of <varname>b</varname>.</para>
+       </callout>
+      </calloutlist>
+     </example>
+ 
+     <para>Before we describe how to create a stencil, we describe how
+     to apply a stencil to an array, yielding values.  To compute the
+     value associated with index position (1,3), the stencil's center
+     is placed at (1,3).  The stencil's
+     <function>upperExtent</function> and
+     <function>lowerExtent</function> functions indicate which &array;
+     values the stencil's function will use.  See <xref
+     linkend="tutorial-array_stencil-doof2d-apply_stencil"></xref>.
+     Applying the stencil's function call
+     <function>operator()</function> yields the computed value.  To
+     compute multiple &array; values, apply a stencil to the array and
+     a domain object: <statement>stencil(b,
+     interiorDomain)</statement>.  This applies the stencil to each
+     position in the domain.  The user must ensure that applying the
+     stencil does not access nonexistent &array; values.</para>
+ 
+     <figure float="1" id="tutorial-array_stencil-doof2d-apply_stencil">
+      <title>Applying a Stencil to an &array;</title>
+      <mediaobject>
+       <imageobject>
+        <imagedata fileref="figures/doof2d.211" format="EPS" align="center"></imagedata>
+       </imageobject>
+       <textobject>
+        <phrase>Apply a stencil to position (1,3) of an array.</phrase>
+       </textobject>
+       <caption>
+        <para>To compute the value associated with index position (1,3)
+        of an array, place the stencil's center, indicated with dashed
+        lines, at the position.  The computation involves the array
+        values covered by the array and delineated by
+        <function>upperExtent</function> and
+        <function>lowerExtent</function>.</para>
+       </caption>
+      </mediaobject>
+     </figure>
+ 
+     <para>To create a stencil object, apply the &stencil; type to a
+     function object class.  For example,
+     <statement>Stencil&lt;DoofNinePt&gt; stencil</statement> declares
+     the <varname>stencil</varname> object.  The function object class
+     must define a function call <function>operator()</function> with a
+     container parameter and index parameters.  The number of index
+     parameters, indicating the stencil's center, must equal the
+     container's dimension.  For example, <type>DoofNinePt</type>
+     defines <methodname>operator()(const C& c, int i, int
+     j)</methodname>.  We templated the container type
+     <varname>C</varname> although this is not strictly necessary.  The
+     two index parameters <varname>i</varname> and <varname>j</varname>
+     ensure the stencil works with two-dimensional containers.  The
+     <methodname>lowerExtent</methodname> indicates how far to the left
+     (or below) the stencil extends beyond its center.  Its parameter
+     indicates a particular dimension.  Index parameters
+     <varname>i</varname> and <varname>j</varname> are in dimension 0
+     and&nbsp;1.  <methodname>upperExtent</methodname> serves an
+     analogous purpose.  The &poomaToolkit; uses these functions when
+     distribution computation among various processors, but it does not
+     use these functions to ensure nonexistent &array; values are not
+     accessed.  Caveat stencil user!</para>
+    </section>
+ 
+ 
+    <section id="tutorial-distributed">
+     <title>Distributed &array; Implementation</title>
+ 
+     <para>A &pooma; program can execute on one or multiple processors.
+     To convert a program designed for uniprocessor execution to a
+     program designed for multiprocessor execution, the programmer need
+     only specify how each container's domain should be split into
+     <quote>patches</quote>.  The &poomaToolkit; automatically
+     distributes the data among the available processors and handles
+     any required communication between processors.</para>
+ 
+     <example id="tutorial-array_distributed-doof2d">
+      <title>Distributed Stencil &array; Implementation of &doof2d;</title>
+      &doof2d-array-distributed;
+      <calloutlist>
+       <callout arearefs="tutorial-array_distributed-doof2d-nuprocessors">
+        <para>The number of processors executing a &pooma; program can
+        be specified at run-time.</para>
+       </callout>
+       <callout arearefs="tutorial-array_distributed-doof2d-layout">
+        <para>The <type>UniformGridPartition</type> declaration
+        specifies how an array's domain will be partition, of split,
+        into patches.  Guard layers are an optimization that can reduce
+        data communication between patches.  The
+        <type>UniformGridLayout</type> declaration applies the
+        partition to the given domain, distributing the resulting
+        patches among various processors.</para>
+       </callout>
+       <callout arearefs="tutorial-array_distributed-doof2d-remote">
+        <para>The <type>MultiPatch</type> &engine; distributes requests
+        for &array; values to the associated patch.  Since a patch may
+        associated with a different processor, its
+        <quote>remote</quote> engine has type
+        <type>Remote&lt;Brick&gt;</type>.  &pooma; automatically
+        distributes the patches among available memories and
+        processors.</para>
+       </callout>
+       <callout
+        arearefs="tutorial-array_distributed-doof2d-first_write">
+        <para>The stencil computation, whether for one processor or
+        multiple processors, is the same.</para>
+       </callout>
+      </calloutlist>
+     </example>
+ 
+     <para>Supporting distributed computation requires only minor code
+     changes.  These changes specify how each container's domain is
+     distributed among the available processors.  The rest of the
+     program, including all the computations, remains the same.  When
+     running, the &pooma; executable interacts with the run-time
+     library to determine which processors are available, distributes
+     the containers' domains, and automatically handles all necessary
+     interprocessor communication.  The same executable runs on one or
+     many processors.  Thus, the programmer can write one program,
+     debugging it on a uniprocessor computer and running it on a
+     supercomputer.</para>
+ 
+     <figure float="1" id="tutorial-array_distributed-doof2d-distributed_model">
+      <title>The &pooma; Distributed Computation Model</title>
+      <mediaobject>
+       <imageobject>
+        <imagedata fileref="figures/distributed.101" format="EPS" align="center"></imagedata>
+       </imageobject>
+       <textobject>
+        <phrase>the &pooma; distributed computation model.</phrase>
+       </textobject>
+       <caption>
+        <para>The &pooma; distributed computation model combines
+        partitioning containers' domains and the computer configuration
+        to create a layout.</para>
+       </caption>
+      </mediaobject>
+     </figure>
+ 
+     <para>&pooma;'s distributed computing model separates container
+     domain concepts from computer configuration concepts.  See <xref
+     linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>.
+     The program indicates how each container's domain will be
+     partitioned.  This process is represented in the upper left corner
+     of the figure.  A user-specified partition specifies how to split
+     the domain into pieces.  For example, the illustrated partition
+     splits the domain into three equal-sized pieces along the
+     x-dimension and two equal-sized pieces along the y-dimension.
+     Thus, the domain is split into <firstterm>patches</firstterm>.
+     The partition also specifies external and internal guard layers.
+     A <firstterm>guard layer</firstterm> is a domain surrounding a
+     patch.  A patch's computation only reads but does not write these
+     values.  An <firstterm>external guard layer</firstterm>
+     conceptually surrounds the entire container domain with boundary
+     values whose presence permits all domain computations to be
+     performed the same way even for values along the domain's edge.
+     An <firstterm>internal guard layer</firstterm> duplicates values
+     from adjacent patches so communication need not occur during a
+     patch's computation.  The use of guard layers is an optimization;
+     using external guard layers eases programming and using internal
+     guard layers reduces communication between processor.  Their use
+     is not required.</para>
+ 
+     <para>The computer configuration of shared memory and processors
+     is determined by the run-time system.  See the upper right portion
+     of <xref
+     linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>.
+     A <firstterm>context</firstterm> is a collection of shared memory
+     and processors that can execute a program or a portion of a
+     program.  For example, a two-processor desktop computer might have
+     memory accessible to both processors so it is a context.  A
+     supercomputer consisting of desktop computers networked together
+     might have as many contexts as computers.  The run-time system,
+     e.g., the Message Passing Interface (&mpi;) Communications Library
+     (FIXME: xref linkend="mpi99", <ulink
+     url="http://www-unix.mcs.anl.gov/mpi/"></ulink>) or the &mm;
+     Shared Memory Library (<ulink
+     url="http://www.engelschall.com/sw/mm/"></ulink>), communicates
+     the available contexts to the executable.  &pooma; must be
+     configured for the particular run-time system.  See <xref
+     linkend="installation-distributed_computing"></xref>.</para>
+ 
+     <para>A <firstterm>layout</firstterm> combines patches with
+     contexts so the program can be executed.  If &distributedtag; is
+     specified, the patches are distributed among the available
+     contexts.  If &replicatedtag; is specified, each set of patches is
+     replicated among each context.  Regardless, the containers'
+     domains are now distributed among the contexts so the program can
+     run.  When a patch needs data from another patch, the &pooma;
+     toolkit sends messages to the desired patch uses a message-passing
+     library.  All such communication is automatically performed by the
+     toolkit with no need for programmer or user input.</para>
+ 
+     <para>FIXME: The two previous paragraphs demonstrate confusion
+     between <quote>run-time system</quote> and <quote>message-passing
+     library</quote>.</para>
+ 
+     <para>Incorporating &pooma;'s distributed computation model into a
+     program requires writing very few lines of code.  <xref
+     linkend="tutorial-array_distributed-doof2d"></xref> illustrates
+     this.  The <varname>partition</varname> declaration creates a
+     <type>UniformGridPartition</type> splitting each dimension of a
+     container's domain into equally-sized
+     <varname>nuProcessors</varname> pieces.  The first
+     <type>GuardLayers</type> argument specifies each patch will have
+     copy of adjacent patches' outermost values.  This may speed
+     computation because a patch need not synchronize its computation
+     with other patches' processors.  Since each value's computation
+     requires knowing its surrounding neighbors, the internal guard
+     layer is one layer deep.  The second <type>GuardLayers</type>
+     argument specifies no external guard layer.  External guard layers
+     simplify computing values along the edges of domains.  Since the
+     program already uses only the interior domain for computation, we
+     do not use this feature.</para>
+ 
+     <para>The <varname>layout</varname> declaration creates a
+     <type>UniformGridLayout</type> layout.  As <xref
+     linkend="tutorial-array_distributed-doof2d"></xref> illustrates,
+     it needs to know a container's domain, a partition, the computer's
+     contexts, and a &distributedtag; or &replicatedtag;.  These
+     comprise <varname>layout</varname>'s three parameters; the
+     contexts are implicitly supplied by the run-time system.</para>
+ 
+     <para>To create a distributed &array;, it should be created using
+     a &layout; object and have a &multipatch; engine.  Prior
+     implementations designed for uniprocessors constructed the
+     container using a &domain; object.  A distributed implementation
+     uses a &layout; object, which conceptually specifies a &domain;
+     object and its distribution throughout the computer.  A
+     &multipatch; engine supports computations using multiple patches.
+     The <type>UniformTag</type> indicates the patches all have the
+     same size.  Since patches may reside on different contexts, the
+     second template parameter is <type>Remote</type>.  Its
+     <type>Brick</type> template parameter specifies the engine for a
+     particular patch on a particular context.  Most distributed
+     programs use <type>MultiPatch&lt;UniformTag, Remote&lt;Brick&gt;
+     &gt;</type> or <type>MultiPatch&lt;UniformTag,
+     Remote&lt;CompressibleBrick&gt; &gt;</type> engines.</para>
+ 
+     <para>The computations for a distributed implementation are
+     exactly the same as for a sequential implementation.  The &pooma;
+     Toolkit and a message-passing library automatically perform all
+     computation.</para>
+ 
+     <para>The command to run the programs is dependent on the run-time
+     system.  To use &mpi; with the Irix&nbsp;6.5 operating system, one
+     can use the <command>mpirun</command> command.  For example,
+     <statement>mpirun -np 9 Doof2d-Array-distributed -mpi
+     --num-patches 3</statement> invokes the &mpi; run-time system with
+     nine processors.  The <statement>-mpi</statement> argument tells
+     the &pooma; executable <command>Doof2d-Array-distributed</command>
+     to use the &mpi; Library.  
+ 
+ HERE
+ 
+     The command <statement>Doof2d-Array-distributed -shmem -np 2
+     --num-patches 2</statement> 
+ 
+     To run <command>Doof2d-Array-distributed</command> with the &mm;
+     Shared Memory Library, use 
+ 
+ HERE</para>
+ 
+ <!-- HERE -->
+ 
+     <para>COMMENT: See <filename
+     class="libraryfile">background.html</filename> for a partial
+     explanation.  A <firstterm>context</firstterm> is a distinct
+     region of memory in some computer.  Execution thread is associated
+     with each context.  One or more different processors can be
+     associated with the same context.</para>
+ 
+     <para>QUESTION: How do &pooma; parallel concepts compare with
+     Fortran&nbsp;D or high-performance Fortran FINISH CITE:
+     {koelbel94:_high_perfor_fortr_handb}?</para>
+ 
+     <para>QUESTION: What does Cheetah do for us?  Must configure with
+     --messaging and Cheetah library must be available.  When running
+     Doof2d benchmark, use --num-patches N.  On LinuxKCC, use
+     '--num-patches p --run-impls 14 --sim-params N 0 1'.  Runtime
+     system must also provide some support.  How do I write about this?
+     What is an example?  How does one install Cheetah?</para>
+ 
+ <!-- HERE -->
+    </section>
+ 
+ 
+    <section id="tutorial-relations">
+     <title>Relations</title>
+ 
+     <para>UNFINISHED</para>
+ 
+    </section>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="concepts">
+    <title>Overview of &pooma; Concepts</title>
+ 
+    <para>Describe the software application layers similar to
+    <filename>papers/SCPaper-95.html</filename> and "Short Tour of
+    &pooma;" in <filename>papers/SiamOO98_paper.ps</filename>.
+    Section&nbsp;2.2, "Why a Framework?," of
+    <filename>papers/pooma.ps</filename> argues why a layered approach
+    eases use.  Section&nbsp;3.1, "Framework Layer Description,"
+    describes the five layers.</para>
+ 
+    <para>FINISH: Write short glossary entries for each of these.</para>
+ 
+    <para>FINISH: Look through the source code to ensure all main
+    concepts are listed.</para>
+ 
+    <para>Here are (preliminary) &pooma; equations:</para>
+ 
+    <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	  orient="port" pgwide="0">
+     <title>&pooma; <quote>Equations</quote></title>
+ 
+     <tgroup cols="1" align="center">
+      <tbody>
+       <row>
+        <entry><para>field = data + materials + centering + layout + mesh</para></entry>
+        </row>
+       <row>
+        <entry><para>map from space to values</para></entry>
+       </row>
+       <row>
+        <entry><para>array = data + layout</para></entry>
+        </row>
+       <row>
+        <entry><para>map from indices to values</para></entry>
+       </row>
+       <row>
+        <entry><para>mesh = layout + origin + spacings</para></entry>
+        </row>
+       <row>
+        <entry><para>distribute domain through physical space</para></entry>
+       </row>
+       <row>
+        <entry><para>layout = domain + partition + layout_tag (distributed/replicated)</para></entry>
+        </row>
+       <row>
+        <entry><para>distribute domain's blocks among processors/contexts</para></entry>
+       </row>
+       <row>
+        <entry><para>partition = blocks + guard layers</para></entry>
+        </row>
+       <row>
+        <entry><para>split domain into blocks</para></entry>
+       </row>
+       <row>
+        <entry><para>domain = newDomain</para></entry>
+        </row>
+       <row>
+        <entry><para>space of permissible indices</para></entry>
+       </row>
+      </tbody>
+     </tgroup>
+    </table>
+ 
+ 
+    <para>FINISH: Following is a first try at describing the &pooma;
+    abstraction layers.  See also paper illustration.</para>
+ 
+    <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	  orient="port" pgwide="0">
+     <title>&pooma; Abstraction Layers</title>
+ 
+     <tgroup cols="1" align="center">
+      <tbody>
+       <row>
+        <entry><para>application program</para></entry>
+       </row>
+       <row>
+        <entry><para>&array; &field; (should have
+        <type>FieldEngine</type> under it)</para></entry>
+       </row>
+       <row>
+        <entry><para>&engine;</para></entry>
+       </row>
+       <row>
+        <entry><para>evaluators</para></entry>
+       </row>
+      </tbody>
+     </tgroup>
+    </table>
+ 
+    <para>FINISH: How does parallel execution fit in?</para>
+ 
+    <para>FINISH: Should we also name and describe each layer?</para>
+ 
+    <section id="concepts-domains">
+     <title>Domains</title>
+ 
+     <blockquote>
+      <attribution>Section&nbsp;4 "Future Improvements in
+      &pooma;&nbsp;II" of
+      <filename>papers/SiamOO98_paper.ps</filename></attribution>
+      <para>
+      A &domain; is a set of discrete points in some space.&hellip;
+      &domain;s provide all of the expected domain calculus
+      capabilities such as subsetting and intersection.
+      </para>
+     </blockquote>
+ 
+     <para>Section&nbsp;3, "Domains and Views," of
+     <filename>papers/iscope98.pdf</filename> describes five types of
+     domains</para>
+    </section>
+ 
+ 
+    <section id="concepts-layouts">
+     <title>Layouts</title>
+ 
+     <para>UNFINISHED</para>
+ 
+     <para>Also describe partitions and guard cells within here.</para>
+ 
+    </section>
+ 
+ 
+    <section id="concepts-meshes">
+     <title>Meshes</title>
+ 
+     <para>UNFINISHED</para>
+    </section>
+ 
+ 
+    <section id="concepts-data_parallel">
+     <title>Data-Parallel Statements</title>
+ 
+     <para>Can we use "An Overview of &pete;" from
+     <filename>papers/PETE_DDJ/ddj_article.html</filename> or is this
+     too low-level?</para>
+ 
+     <para>Section&nbsp;3.2.1 of <filename>papers/pooma.ps</filename>
+     gives a simple example of data-parallel expression.  It also has a
+     paragraph introducing data-parallel operations and selecting
+     subsets of domains.  Section&nbsp;3.4 describes the Chained
+     Expression Object (<acronym>CEO</acronym>), apparently a precursor
+     of &pete;.  Regardless, it provides some motivation and
+     introductory material.</para>
+ 
+     <para>From Section&nbsp;4 of
+     <filename>papers/SiamOO98_paper.ps</filename>:</para>
+ 
+     <para>This version of &pete; reduces compile time of user codes
+     and utilizes compile-time knowledge of expression &domain;s for
+     better optimization.  For example, more efficient loops for
+     evaluating an expression can be generated if &pete; knows that the
+     &domain; has unit stride in memory.</para>
+ 
+     <para>Section&nbsp;4, "Expressions and Evaluators", of
+     <filename>papers/iscope98.pdf</filename> has a good explanation of
+     &pooma;&nbsp;II's expression trees and expression engines.</para>
+ 
+     <para>COMMENT: <filename
+ 			     class="libraryfile">background.html</filename> has some related
+     &pete; material.</para>
+    </section>
+ 
+    <section id="concepts-containers">
+     <title>Containers</title>
+ 
+     <section id="concepts-containers-array">
+      <title>&array;</title>
+ 
+      <blockquote>
+       <attribution>Section&nbsp;4 "Future Improvements in
+      &pooma;&nbsp;II" of
+      <filename>papers/SiamOO98_paper.ps</filename></attribution>
+       <para>
+      An &array; can be thought of as a map from one &domain; to
+      another.&hellip; &array;s depend only on the interface of
+      &domain;s.  Thus, a subset of view of an &array; can be
+      manipulated in all the same ways as the original &array;.
+      &array;s can perform indirect addressing because the output
+      &domain; one one &array; can be used as the input &domain; of
+      another &array;.  &array;s also provide individual element
+      access.</para>
+      </blockquote>
+ 
+ 
+      <para>
+       (unformatted) From
+       <filename>papers/GenericProgramming_CSE/dubois.html</filename>:
+       The &pooma; &array; concept provides an example of how these
+       generic-programming features can lead to flexible and efficient
+       code. An Array maps a fairly arbitrary input domain to an
+       arbitrary range of outputs. When used by itself, an &array;
+       object <varname>A</varname> refers to all of the values in its
+       domain. Element-wise mathematical operations or functions can be
+       applied to an array using straightforward notation, like A + B
+       or sin(A). Expressions involving Array objects are themselves
+       Arrays. The operation A(d), where d is a domain object that
+       describes a subset of A's domain, creates a view of A that
+       refers to that subset of points. Like an array expression, a
+       view is also an Array. If d represents a single point in the
+       domain, this indexing operation returns a single value from the
+       range. Equivalently, one can index an N-dimensional Array by
+       specifying N indices, which collectively specify a single point
+       in the input domain: A(i1, i2, ..., iN).</para>
+ 
+      <para>The &pooma; multi-dimensional Array concept is similar to
+       the Fortran 90 array facility, but extends it in several
+       ways. Both &pooma; and Fortran arrays can have up to seven
+       dimensions, and can serve as containers for arbitrary
+       types. Both support the notion of views of a portion of the
+       array, known as array sections in F90. The &pooma; Array concept
+       supports more complex domains, including bounded, continuous
+       (floating-point) domains. Furthermore, Array indexing in &pooma;
+       is polymorphic; that is, the indexing operation X(i1,i2) can
+       perform the mapping from domain to range in a variety of ways,
+       depending on the particular type of the Array being
+       indexed.</para>
+ 
+      <para>Fortran arrays are dense and the elements are arranged
+       according to column-major conventions. Therefore, X(i1,i2)
+       refers to element number i1-1+(i2-1)*numberRowsInA. However, as
+       Fig. 1 shows, Fortran-style "Brick" storage is not the only
+       storage format of interest to scientific programmers. For
+       compatibility with C conventions, one might want to use an array
+       featuring dense, row-major storage (a C-style Brick). To save
+       memory, it might be advantageous to use an array that only
+       stores a single value if all its element values are the
+       same. Other sparse storage schemes that only store certain
+       values may also be desirable. To exploit parallelism, it is
+       convenient for an array's storage to be broken up into patches,
+       which can be processed independently by different CPUs. Finally,
+       one can imagine an array with no data at all. For example, the
+       values can be computed from an expression involving other
+       arrays, or analytically from the indices.
+      </para>
+ 
+      <para>The &pooma; &array; Class Template</para>
+ 
+      <para>Next we describe &pooma;'s model of the Array concept, the
+      Array class template. The three most important requirements from
+      the point of view of overall design are: (1) arbitrary domain,
+      (2) arbitrary range, and (3) polymorphic indexing. These express
+      themselves in the template parameters for the &pooma; Array
+      class. The template
+      <programlisting>
+      template &lt;int Dim, class T = double, class EngineTag = Brick&gt;
+      class Array;
+      </programlisting>
+      is a specification for creating a set of classes all named
+      Array. The template parameters Dim, T, and EngineTag determine
+      the precise type of the Array. Dim represents the dimension of
+      the array's domain. T gives the type of array elements, thereby
+      defining the output range of the array. EngineTag specifies the
+      the manner of indexing and types of the indices.</para>
+ 
+      <para>End From
+      <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
+ 
+      <para>Section&nbsp;2, "Arrays and Engines," of
+      <filename>papers/iscope98.pdf</filename> describes both &array;s
+      and &engine;s.  This may or may not duplicate the material in
+      <filename>papers/GenericProgramming_CSE/dubois.html</filename>.</para>
+ 
+      <section id="concepts-containers-array-views">
+       <title>Views of &array;s</title>
+ 
+       <para>Section&nbsp;3, "Domains and Views," of
+       <filename>papers/iscope98.pdf</filename> motivates the need for
+       views:
+        <blockquote>
+ 	<para>One of the primary uses of domains is to specify
+ 	subsections of &array; objects.  Subarrays are a common
+ 	feature of array classes; however, it is often difficult to
+ 	make such subarrays behave like first-class objects.  The
+ 	&pooma;&nbsp;II engine concept provides a clean solution to
+ 	this problem: subsetting an &array; with a domain object
+ 	creates a new &array; that has a view engine.</para>
+        </blockquote>
+       </para>
+      </section>
+     </section>
+ 
+     <section id="concepts-containers-field">
+      <title>&field;</title>
+ 
+      <para>QUESTION: Do we include boundary conditions here?
+ 
+      FINISH: Do we have an example that shows something not possible
+      with &array;?</para>
+ 
+      <para>Describe and illustrate multi-material and
+      multivalue?</para>
+ 
+      <para>ADD: description of meshes and guard layers.</para>
+ 
+     </section>
+ 
+ 
+     <section id="concepts-containers-matrix">
+      <title><type>TinyMatrix</type></title>
+ 
+      <para>Section&nbsp;3.2.2 of
+      <filename>papers/pooma.ps</filename> describes &vector;s and
+      matrix classes.</para>
+     </section>
+    </section>
+ 
+    <section id="concepts-engines">
+     <title>Engines</title>
+ 
+     <para>(unformatted) From
+     <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
+ 
+     <para>The Engine Concept</para>
+ 
+     <para>To implement polymorphic indexing, the Array class defers
+     data storage and data lookup to an engine object. The requirements
+     that the Array template places on its engine provide the
+     definition for the Engine concept. We'll describe these by
+     examining a simplified version of the Array template, shown in
+     Fig. 2.</para>
+ 
+     <para>First, the Array class determines and exports (makes
+     Engine_t part of Array's public interface) the type of the engine
+     class that it will use:
+     <programlisting>
+     typedef Engine&lt;Dim, T, EngineTag&gt; Engine_t;
+     </programlisting>
+     This statement declares Engine_t to be an alias for the type
+     Engine&lt;Dim,T,EngineTag&gt;. This is the first requirement
+     placed on engine classes: they must be specializations of a
+     general Engine template whose template parameters are identical to
+     those of Array. Next, the Array template determines the type of
+     scalar arguments (indices) to be used in operator(), the function
+     that implements &pooma;'s Fortran-style indexing syntax X(i1,i2):
+     <programlisting>
+     typedef typename Engine_t::Index_t Index_t;
+     </programlisting>
+     This statement defines another type alias:
+     Array&lt;Dim,T,EngineTag&gt;::Index_t is simply an alias for
+     Engine_t::Index_t. Engine_t::Index_t is a qualified name, which
+     means that the type Index_t is found in the class Engine_t. This
+     is the second requirement for the Engine concept: the class
+     Engine_t must define a public type called Index_t. This line will
+     not compile if that definition is not supplied. This indirection
+     is one of the ways that &pooma; supports polymorphic indexing. If
+     the Engine works with a discrete integer domain, it defines its
+     Index_t to be an integral type. If the Engine works in a
+     continuous domain, it defines its Index_t to be a floating-point
+     type.</para>
+ 
+     <para>The data lookup is performed in the operator() function. We
+     see that Array simply passes the indices on to its engine
+     object. Thus, we have the third requirement for the Engine
+     concept: it must provide a version of operator() that takes Dim
+     values of type Index_t.</para>
+ 
+     <para>Simply passing the indices on to the engine object may seem
+     odd. After all, engine(i,j) looks like we're just indexing another
+     array. There are several advantages to this extra level of
+     indirection. The Array class is as faithful a model of the Array
+     concept as possible, while the Engine class is a low-level
+     interface to a user-defined data source. As a result, Array has a
+     wide variety of constructors for user convenience, while engines
+     have but a few. Array supports a wide variety of overloaded
+     operator() functions for view creation and indexing. Engines
+     support indexing only. Array does not have direct access to the
+     data, which is managed by the engine object. Finally, Array has a
+     wide variety of overloaded mathematical operators and functions,
+     and works with the Portable Expression Template Engine (PETE) [4]
+     to provide efficient evaluation of Array expressions. Engines have
+     no such support. In general, Array is much more complex and
+     feature-laden than Engine. This is the prime advantage of the
+     separation of interface and implementation: Array only has to be
+     implemented once by the &pooma; developers. Engines are simple
+     enough to be written by users and plugged directly into the Array
+     framework.</para>
+ 
+     <para>Figure 3 illustrates the "Brick" specialization of the
+     Engine template, which implements Fortran-style lookup into a
+     block of memory. First, there is the general Engine template,
+     which is empty as there is no default behavior for an unknown
+     EngineTag. The general template is therefore not a model for the
+     Engine concept and Array classes attempting to use it will not
+     compile. Next, there is the definition of the Brick class, a
+     policy tag whose sole purpose is to select a particular
+     specialization of the Engine template. Finally, there is the
+     partial specialization of the Engine template. Examining its body,
+     we see the required Index_t typedef and the required operator(),
+     which follows the Fortran prescription for generating an offset
+     into the data block based on the row, column, and the number of
+     rows. All of the requirements are met, so the Brick-Engine class
+     is a model of the Engine concept.</para>
+ 
+     <para>End From
+     <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
+ 
+     <para>(unformatted) From
+     <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
+ 
+     <para>Compile-time Versus Run-Time Polymorphism</para>
+ 
+     <para>Encapsulating the indexing in an Engine class has important
+     advantages, both in terms of flexibility and efficiency. To
+     illustrate this point, we introduce the PolarGaussian-Engine
+     specialization in Fig. 4. This is an analytic engine that
+     calculates its values directly from its inputs. Unlike the
+     Brick-Engine, this engine is "indexed" with data of the same type
+     as its output: it maps a set of T's to a single T. Therefore, the
+     Index_t typedef selects T as the index type, as opposed to the int
+     in the Brick-Engine specialization. The operator() function also
+     differs in that it computes the return value according to an
+     analytic formula.</para>
+ 
+     <para>Both Engine&lt;Dim,T,Brick&gt; and
+     Engine&lt;Dim,T,PolarGaussian&gt; can be plugged in to an Array by
+     simply varying the Array's EngineTag. This is possible despite the
+     fact that the two classes exhibit dramatically different behavior
+     because they are both models of the Engine concept.</para>
+ 
+     <para>Notice that we have achieved polymorphic indexing without
+     the use of inheritance or virtual functions. For instance,
+     consider the following code snippet:
+     <programlisting>
+     Array<2, double, Brick> a;
+     Array<2, double, PolarGaussian> b;
+ 
+     double x = a(2, 3);     // x = a.engine.data[2 + 3 * a.engine.numRows];
+     double y = b(2.0, 3.0); // y = exp(-(2.0*2.0+3.0*3.0) / b.engine.delta);
+     </programlisting>
+     The data lookup functions for the two Arrays perform completely
+     different operations. Since this is accomplished using static
+     types, it is known as compile-time polymorphism. Moreover,
+     everything is known at compile time, so the functions are fully
+     inlined and optimized, thereby yielding code equivalent to that
+     shown in the comments above.</para>
+ 
+     <para>The flexibility and efficiency of compile-time polymorphism
+     cannot be duplicated with a run-time implementation. To illustrate
+     this point, in Fig. 5, we re-implement our Array concept using the
+     classic Envelope-Letter pattern [5], with the array class,
+     RTArray, being the envelope and the run-time-engine, RTEngine,
+     being the letter. RTArray defers data lookup to the engine object
+     by invoking the engine's functions through a pointer to the
+     RTEngine base class. Figure 6 illustrates the RTEngine base class
+     and Fig. 7 illustrates two descendants: RTBrick and
+     RTPolarGaussian.</para>
+ 
+     <para>The run-time implementation provides the same basic
+     functionality as the compile-time implementation, but it is not as
+     flexible or as efficient. It lacks flexibility in that the return
+     type of the indexing operation must be specified in the RTEngine
+     base class and in the RTArray class. Thus, in Figs. 5 and 6,we see
+     versions of RTArray::operator() and RTEngine::index functions that
+     take both int's and T's. If the programmer wants to add another
+     index-type option, these classes must be modified. This is a
+     violation of the open-closed principle proposed by Meyer
+     [6]. Also, since RTEngine descendants will usually only implement
+     one version of index, we cannot make RTEngine an abstract base
+     class. Instead, we have the default versions of index throw an
+     exception. Thus, compile-time error checking is
+     weakened. Furthermore, since indexing is done via a virtual
+     function call, it will almost never be inlined, which is not
+     acceptable in most scientific applications.</para>
+ 
+     <para>There are advantages to the Envelope-Letter approach. First,
+     all RTArray objects have the same type, allowing them to be stored
+     in homogeneous collections. This can simplify the design of some
+     applications. Second, RTArray objects can change their engines at
+     runtime, and thus effectively change their types on the fly??this
+     is the primary reason for using the Envelope-Letter idiom, and can
+     be very important in some applications.</para>
+ 
+     <para>For most scientific applications, however, these issues are
+     minor, and maximum performance for array indexing is of paramount
+     importance. Our compile-time approach achieves this performance
+     while providing the desired polymorphic indexing.</para>
+ 
+     <para>From Section&nbsp;4 of
+     <filename>papers/SiamOO98_paper.ps</filename>:</para>
+ 
+     <para>The &array; class is templated on an &engine; type that
+     handles the actual implementation of the mapping from input to
+     output.  Thus, the &array; interface features are completely
+     separate from the implementation, which could be a single &c;
+     array, a function of some kind or some other mechanism.  This
+     flexibility allows an expression itself to be viewed through the
+     &array; interface.  Thus, one can write something like
+     <programlisting>
+     foo(A*B+C);
+     </programlisting> where <varname>A</varname>, <varname>B</varname> and
+     <varname>C</varname> are &array;s and <function>foo</function> is
+     a function taking an &array; as an argument.  The expression
+     <statement><varname>A</varname>*<varname>B</varname>+<varname>C</varname></statement>
+     will only be evaluated by the expression engine as needed by
+     <function>foo</function>.</para>
+ 
+     <para>In fact, one can even write &engine;s which are wrappers
+     around external data structures created in non-&pooma; codes and
+     know to manipulate these structures.  Once this is done, the
+     external entities have access to the entire &array; interface and
+     can utilize all of the powerful features of
+     &pooma;&nbsp;II.</para>
+ 
+     <para>Section&nbsp;2, "Arrays and Engines," of
+     <filename>papers/iscope98.pdf</filename> describes both &array;s
+     and &engine;s.  This may or may not duplicate the material in
+     <filename>papers/GenericProgramming_CSE/dubois.html</filename>.</para>
+ 
+     <para>Section&nbsp;4, "Expressions and Evaluators", of
+     <filename>papers/iscope98.pdf</filename> has a good explanation of
+     &pooma;&nbsp;II's expression trees and expression engines.</para>
+ 
+     <variablelist>
+      <varlistentry><term><type>MultiPatch</type> Engine</term>
+       <listitem><para>From <filename
+       class="libraryfile">README</filename>: To actually use multiple
+       contexts effectively, you need to use the MultiPatch engine with
+       patch engines that are Remote engines.  Then the data will be
+       distributed across multiple contexts instead of being copied on
+       every context.  See the files in example/Doof2d for a simple
+       example that creates a MultiPatch array that can be distributed
+       across multiple contexts and performs a stencil computation on
+       that array.</para></listitem>
+      </varlistentry>
+     </variablelist>
+    </section>
+ 
+ 
+    <section id="concepts-relations">
+     <title>Relations</title>
+ 
+     <para>UNFINISHED</para>
+    </section>
+ 
+ 
+    <section id="concepts-stencils">
+     <title>Stencils</title>
+ 
+     <para>Section&nbsp;3.5.4, "Stencil Objects," of
+     <filename>papers/pooma.ps</filename> provides a few uses of
+     stencils.</para>
+ 
+     <para>Section&nbsp;5, "Performance," of
+     <filename>papers/iscope98.pdf</filename> motivates and explains
+     stencils.</para>
+    </section>
+ 
+ 
+    <section id="concepts-contexts">
+     <title>Contexts</title>
+ 
+     <blockquote>
+      <attribution><filename
+      class="libraryfile">background.html</filename></attribution>
+      <para>In order to be able to cope with the variations in machine
+      architecture noted above, &pooma;'s distributed execution model
+      is defined in terms of one or more contexts, each of which may
+      host one or more threads. A context is a distinct region of
+      memory in some computer. The threads associated with the context
+      can access data in that memory region and can run on the
+      processors associated with that context. Threads running in
+      different contexts cannot access memory in other contexts.</para>
+ 
+      <para>A single context may include several physical processors,
+      or just one. Conversely, different contexts do not have to be on
+      separate computers&mdash;for example, a 32-node SMP computer could
+      have up to 32 separate contexts. This release of &pooma; only
+      supports a single context for each application, but can use
+      multiple threads in the context on supported platforms. Support
+      for multiple contexts will be added in an upcoming
+      release.</para>
+     </blockquote>
+    </section>
+ 
+ 
+    <section id="concepts-utilities">
+     <title>Utility Types: ???TITLE??</title>
+ 
+     <section id="concepts-containers-vector">
+      <title>&vector;</title>
+ 
+      <para>Section&nbsp;3.2.2 of
+      <filename>papers/pooma.ps</filename> describes &vector;s and
+      matrix classes.</para>
+     </section>
+ 
+    </section>
+   </chapter>
+ 
+ 
+   <chapter id="sequential">
+    <title>Writing Sequential Programs</title>
+ 
+    <para>UNFINISHED</para>
+ 
+    <section id="sequential-benchmarks">
+     <title>&benchmark; Programs</title>
+ 
+     <para>Define a &benchmark; program vs. an example or an
+     executable.  Provide a short overview of how to run these
+     programs.  Provide an overview of how to write these programs.
+     See <filename
+ 		  class="headerfile">src/Utilities/Benchmark.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="sequential-inform">
+     <title>Using <type>Inform</type>s for Output</title>
+ 
+     <para>UNFINISHED</para>
+    </section>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="parallel">
+    <title>Writing Distributed Programs</title>
+ 
+    <para>Discuss the distributed model and guard cells.  See <filename
+    class="libraryfile">docs/parallelism.html</filename>.</para>
+ 
+    <para>Does any of the parallel implementation described in
+    <filename>papers/SCPaper-95.html</filename> still apply?</para>
+ 
+    <para>?Tuning program for maximize parallel performance?</para>
+ 
+    <para>external references to &mpi; and threads</para>
+ 
+    <para>QUESTION: Are there interesting, short parallel programs in
+    any &mpi; book that we can convert to &pooma;?</para>
+ 
+    <section>
+     <title>Layouts</title>
+ 
+     <para>An out-of-date description can be found in Section&nbsp;3.3,
+     especially 3.3.2, of <filename>papers/pooma.ps</filename>
+     describes the global/local interactions and parallel abstraction
+     layers.</para>
+    </section>
+ 
+    <section>
+     <title>Parallel Communication</title>
+ 
+     <para>An out-of-date description can be found in
+      Section&nbsp;3.3.3 of <filename>papers/pooma.ps</filename></para>
+    </section>
+ 
+    <section>
+     <title>Using Threads</title>
+ 
+     <para>QUESTION: Where do threads fit into the manual?  Do threads
+     even work?</para>
+ 
+     <para>From Section&nbsp;4, of
+     <filename>papers/SiamOO98_paper.ps</filename></para>
+ 
+     <para>&pooma;&nbsp;II will make use of a new parallel run-time
+     system called &smarts; that is under development at the ACL.
+     &smarts; supports lightweight threads, so the evaluator will be
+     able to farm out data communication tasks and the evaluation of
+     subsets of an expression to multiple threads, thus increasing the
+     overlap of communication and computation.  Threads will also be
+     available at the user level for situations in which a
+     task-parallel approach is deemed appropriate.</para>
+    </section>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="under_the_hood">
+    <title>Under the Hood: How &pooma; Works</title>
+ 
+    <para>from point of view of &cc; interpreter</para>
+ 
+    <section id="under_the_hood-PETE">
+     <title>&pete;</title>
+ 
+     <para>Use the material in
+     <filename>papers/PETE_DDJ/ddj_article.html</filename>, which gives
+     example code and descriptions of how the code works.</para>
+ 
+     <para>See material in <filename
+     class="libraryfile">background.html</filename>'s <quote>Expression
+     Templates.</quote></para>
+    </section>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="debugging_profiling">
+    <title>Debugging and Profiling &pooma; Programs</title>
+ 
+    <para>UNFINISHED</para>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="example">
+    <title>Example Program: Jacobi Solver</title>
+ 
+    <para>QUESTION: Is this chapter necessary?  Do we have enough
+    existing source code to write this chapter?</para>
+ 
+   </chapter>
+  </part>
+ 
+  <part id="reference">
+   <title>&pooma; Reference Manual</title>
+ 
+   <chapter id="tmp_ref">
+    <title>TMP: This Chapter Holds These Comments But Will Be Removed</title>
+ 
+    <para>For each template parameter need to describe the constraints
+    on it.</para>
+ 
+    <para>Remove this section when the following concerns have been
+    addressed.</para>
+ 
+    <para>Add a partintro explaining file suffixes such as <filename
+    class="headerfile">.h</filename>, <filename
+    class="libraryfile">.cpp</filename>, <filename
+    class="libraryfile">.cmpl.cpp</filename>, <filename
+    class="libraryfile">.mk</filename>, <filename
+    class="libraryfile">.conf</filename>.  Should we also explain use
+    of <literal>inline</literal> even when necessary and the template
+    model, <!-- FIXME: s/literal/keyword/ --> e.g., including <filename
+    class="libraryfile">.cpp</filename> files.</para>
+ 
+    <para>QUESTION: What are the key concepts around which to organize
+    the manual?</para>
+ 
+    <para>QUESTION: What format should the manual use?</para>
+ 
+    <blockquote>
+     <attribution>Musser, Derge, and Sanai, &sect;20.0.</attribution>
+     <para>It is important to state the requirements on the components
+     as generally as possible.  For example, instead of saying
+     <quote>class <type>X</type> must define a member function
+     <function>operator++()</function>,</quote> we say <quote>for any
+     object&nbsp;<varname>x</varname> of type&nbsp;<type>X</type>,
+     <classname>++x</classname> is defined.</quote></para>
+    </blockquote>
+   </chapter>
+ 
+ 
+   <chapter id="class_structure_ref">
+    <title>A Typical &pooma; Class</title>
+ 
+    <variablelist>
+     <title>Class Member Notation</title>
+     <varlistentry>
+      <term><fieldsynopsis>
+        <varname>*_t</varname>
+       </fieldsynopsis>
+      </term>
+      <listitem>
+       <para>type within a class.  QUESTION: What is the &cc; name for
+       this?</para>
+      </listitem>
+     </varlistentry>
+     <varlistentry>
+      <term><fieldsynopsis>
+        <varname>*_m</varname>
+       </fieldsynopsis>
+      </term>
+      <listitem>
+       <para>data member</para>
+      </listitem>
+     </varlistentry>
+    </variablelist>
+ 
+    <variablelist>
+     <title>&pooma; Class Vocabulary</title>
+     <varlistentry>
+      <term>component</term>
+      <listitem>
+       <para>one of several values packaged together.  For example, a
+       three-dimensional vector has three components, i.e., three
+       values.</para>
+      </listitem>
+     </varlistentry>
+     <varlistentry>
+      <term>element-wise</term>
+      <listitem>
+       <para>applied to each element in the group, e.g., an array</para>
+      </listitem>
+     </varlistentry>
+     <varlistentry>
+      <term>reduction</term>
+      <listitem>
+       <para>repeated application of a binary operator to all elements,
+       yielding one value</para>
+      </listitem>
+     </varlistentry>
+     <varlistentry>
+      <term>tag</term>
+      <listitem>
+       <para>an enumerated value indicating inclusion in a particular
+       semantic class.  The set of values need not be explicitly
+       declared.</para>
+      </listitem>
+     </varlistentry>
+    </variablelist>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="installing_configuring_ref">
+    <title>Installing and Configuring &pooma;</title>
+ 
+    <orderedlist>
+     <listitem>
+      <para>Installing &pooma;.</para>
+     </listitem>
+     <listitem>
+      <para>Requirements for configuration files.</para>
+     </listitem>
+    </orderedlist>
+ 
+    <para>Include descriptions of using &smarts;, &cheetah;, &tau;,
+    &pdt;.</para>
+ 
+    <para>QUESTION: Does it install on windows and on mac?  If so, what
+    are the instructions?  See also <filename
+    class="libraryfile">INSTALL.{mac,unix,windows}.</filename></para>
+ 
+    <para><filename class="libraryfile">README</filename> has some
+    information on &cheetah; and threads in the <quote>Message-Based
+    Parallelism</quote> section.</para>
+ 
+    <para>Which additional packages are necessary and when?</para>
+ 
+    <para>What configure options should we list?  See <filename
+    class="libraryfile">configure</filename>.  Be sure to list
+    debugging option and how its output relates to <filename
+    class="libraryfile">config/LINUXgcc.suite.mk</filename>.</para>
+ 
+    <para><filename class="directory">config/arch</filename> has files
+    for (OS, compiler) pairs.  Explain how to modify a configuration
+    file.  List requirements when making a new configuration file (low
+    priority).</para>
+ 
+    <para><filename
+    class="libraryfile">config/LINUXgcc.suite.mk</filename> has output
+    from <filename class="libraryfile">configure</filename>.  Useful to
+    relate to configuration files and <filename
+    class="libraryfile">configure</filename>'s debugging output.</para>
+    
+   </chapter>
+ 
+ 
+   <chapter id="compilation_ref">
+    <title>Compilation and &make; Files</title>
+ 
+    <para>We assume Gnu make.  Do we know what assumptions are made?</para>
+ 
+    <para>How do all these files interact with each other?  Ala a make
+    interpreter, give an example of which files are read and
+    when.</para>
+ 
+    <variablelist>
+     <varlistentry><term><filename
+     class="libraryfile">config/Shared/README.make</filename></term>
+     <listitem><para>This has short descriptions of many files,
+     especially in <filename
+     class="directory">config/Shared</filename>.</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">makefile</filename></term>
+     <listitem><para>These appear throughout all directories.  What are
+     the equivalences classes and what are their
+     parts?</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">include.mk</filename></term>
+     <listitem><para>What does this do?  Occurs in many directories:
+     when?  Template seems to be <filename
+     class="libraryfile">config/Shared/include2.mk</filename>.</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">subdir.mk</filename></term>
+     <listitem><para>list of subdirectories; occurs in several
+     directories: when?  <filename
+     class="libraryfile">src/subdir.mk</filename> is a good
+     example.</para></listitem>
+     </varlistentry>
+     <varlistentry>
+      <term><filename class="libraryfile">objfile.mk</filename></term>
+      <listitem>
+       <para>list of object files to construct, presumably from
+       <filename>*.cmpl.cpp</filename> files.
+       <filename>src/Utilities/objfile.mk</filename> is an
+       example.</para>
+      </listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">config/Shared/rules.mk</filename></term>
+     <listitem><para>most compiler rules</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">config/head.mk</filename></term>
+     <listitem><para>read at beginning of each
+     makefile?</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">config/Shared/tail.mk</filename></term>
+     <listitem><para>read at end of each makefile?</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">config/Shared/variables.mk</filename></term>
+     <listitem><para>Is this used?</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">config/Shared/compilerules.mk</filename></term>
+     <listitem><para>table of origin and target suffixes and commands
+     for conversion</para></listitem>
+     </varlistentry>
+ 
+    </variablelist>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="arrays_ref">
+    <title>&array;s</title>
+ 
+    <para>Include <filename
+ 			   class="headerfile">src/Pooma/Arrays.h</filename> to use &array;s.
+    The implementation source code is in <filename
+ 						  class="directory">src/Array</filename>.</para>
+ 
+    <para>FINISH: Define an array.  Introduce its parts.</para>
+ 
+    <para>ADD: some mention of the maximum supported number of
+    dimensions somewhere.</para>
+ 
+    <section id="arrays_ref-array">
+     <title>The &array; Container</title>
+ 
+     <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	   orient="port" pgwide="0">
+      <title>Template Parameters</title>
+ 
+      <tgroup cols="2" align="left">
+       <thead>
+        <row>
+ 	<entry>Parameter</entry>
+ 	<entry>Interpretation</entry>
+        </row>
+       </thead>
+       <tbody>
+        <row>
+ 	<entry><varname>Dim</varname></entry>
+ 	<entry><para>dimension</para></entry>
+        </row>
+        <row>
+ 	<entry><varname>T</varname></entry>
+ 	<entry><para>array element type</para></entry>
+        </row>
+        <row>
+ 	<entry>EngineTag</entry>
+ 	<entry><para>type of computation engine object</para></entry>
+        </row>
+       </tbody>
+      </tgroup>
+     </table>
+ 
+     <para>QUESTION: How do I introduce class type definitions, when
+     they are used, i.e., compile-time or run-time, and when
+     programmers should use them?</para>
+ 
+     <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	   orient="port" pgwide="0">
+      <title>Compile-Time Types and Values</title>
+ 
+      <tgroup cols="2" align="left">
+       <thead>
+        <row>
+ 	<entry>Type or Value</entry>
+ 	<entry>Interpretation</entry>
+        </row>
+       </thead>
+       <tbody>
+        <row>
+ 	<entry><fieldsynopsis><varname>This_t</varname></fieldsynopsis></entry>
+ 	<entry><para>the &array; object's type</para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>Engine_t</varname></fieldsynopsis></entry>
+ 	<entry><para>the &array; object's engine's type</para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>EngineTag_t</varname></fieldsynopsis></entry>
+ 	<entry><para>indication of engine's category</para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>Element_t</varname></fieldsynopsis></entry>
+ 	<entry><para>the type of the array elements, i.e., <type>T</type></para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>ElementRef_t</varname></fieldsynopsis></entry>
+ 	<entry><para>the type of a reference to an array element,
+ 	i.e., <type>T&</type>.  Equivalently, the type to write to a
+ 	single element.</para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>Domain_t</varname></fieldsynopsis></entry>
+ 	<entry><para>the array's domain's type, i.e., the type of the
+ 	union of all array indices</para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>Layout_t</varname></fieldsynopsis></entry>
+ 	<entry><para>unknown</para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>dimensions</varname></fieldsynopsis></entry>
+ 	<entry><para>integer equalling the number of dimensions, i.e.,
+ 	<varname>Dim</varname></para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>rank</varname></fieldsynopsis></entry>
+ 	<entry><para>integer equalling the number of dimensions, i.e.,
+ 	<varname>Dim</varname>; a synonym for <fieldsynopsis>
+ 	   <varname>dimensions</varname></fieldsynopsis></para></entry>
+        </row>
+       </tbody>
+      </tgroup>
+     </table>
+ 
+     <section id="arrays_ref-array-constructors">
+      <title>Constructors and Destructors</title>
+ 
+      <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	    orient="port" pgwide="0">
+       <title>Constructors and Destructors</title>
+ 
+       <tgroup cols="2" align="left">
+        <thead>
+ 	<row>
+ 	 <entry>Function</entry>
+ 	 <entry>Effect</entry>
+ 	</row>
+        </thead>
+        <tbody>
+ 	<row>
+ 	 <entry>
+           <constructorsynopsis>
+ 	   <methodname>Array</methodname>
+ 	   <void></void>
+ 	  </constructorsynopsis>
+          </entry>
+ 	 <entry><para>Creates an array that will be resized
+ 	 later.</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>
+           <constructorsynopsis>
+ 	   <methodname>Array</methodname>
+ 	   <methodparam><modifier>const</modifier> <type>Engine_t&</type>
+ 	    <parameter>engine</parameter></methodparam>
+ 	  </constructorsynopsis>
+          </entry>
+ 	 <entry><para>Creates an array with an engine equivalent to
+          the <parameter>engine</parameter>.  This array will have the
+          same values as <parameter>engine</parameter>.  QUESTION: Why
+          would a user every want to use this
+          constructor?</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>
+           <constructorsynopsis>
+ 	   <methodname>Array</methodname>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type> Engine&lt;Dim2, T2, EngineTag2&gt;&</type>
+ 	    <parameter>engine</parameter>
+ 	   </methodparam>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type>Initializer&</type> <parameter>init</parameter>
+ 	   </methodparam>
+ 	  </constructorsynopsis>
+          </entry>
+ 	 <entry><para>What does this do?</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>ADD ALL CONSTRUCTORS AND DESTRUCTORS.</entry>
+ 	</row>
+        </tbody>
+       </tgroup>
+      </table>
+     </section>
+ 
+ 
+     <section id="arrays_ref-array-initializers">
+      <title>Initializers</title>
+ 
+      <para>Add a table.</para>
+     </section>
+ 
+ 
+     <section id="arrays_ref-array-access">
+      <title>Element Access</title>
+ 
+      <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	    orient="port" pgwide="0">
+       <title>&array; Element Access</title>
+ 
+       <tgroup cols="2" align="left">
+        <thead>
+ 	<row>
+ 	 <entry>Function</entry>
+ 	 <entry>Effect</entry>
+ 	</row>
+        </thead>
+        <tbody>
+ 	<row>
+ 	 <entry>
+           <methodsynopsis>
+ 	   <type>Element_t</type> <methodname>read</methodname>
+ 	   <void></void>
+ 	  </methodsynopsis>
+          </entry>
+ 	 <entry><para>unknown: See line 1839.</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>
+           <methodsynopsis>
+ 	   <type>Element_t</type> <methodname>read</methodname>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type>Sub1&</type> <parameter>s1</parameter>
+ 	   </methodparam>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type> Sub2&</type> <parameter>s2</parameter>
+ 	   </methodparam>
+ 	  </methodsynopsis>
+          </entry>
+ 	 <entry><para>How does the version with template parameters,
+          e.g., <type>Sub1</type> differ from the <type>int</type>
+          version?</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>
+           <methodsynopsis>
+ 	   <type>Element_t</type> <methodname>operator()</methodname>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type>Sub1&</type> <parameter>s1</parameter>
+ 	   </methodparam>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type>Sub2&</type> <parameter>s2</parameter>
+ 	   </methodparam>
+ 	  </methodsynopsis>
+          </entry>
+ 	 <entry><para>How does this differ from <statement>read(const
+           Sub1& s1, const Sub2& s2)</statement>?  </para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>ADD ALL <function>read</function>s and
+ 	 <function>operator()</function>s.</entry>
+ 	</row>
+        </tbody>
+       </tgroup>
+      </table>
+     </section>
+ 
+ 
+     <section id="arrays_ref-array-component">
+      <title>Component Access</title>
+ 
+      <para>When an array stores elements having components, e.g., an
+      array of vectors, tensors, or arrays, the
+      <methodname>comp</methodname> returns an array consisting of the
+      specified components.  The original and component array share the
+      same engine so changing the values in one affects values in the
+      other.</para>
+ 
+      <para>For example, if &n; &times; &n; array <varname>a</varname>
+      consists of three-dimensional real-valued vectors,
+      <statement>a.comp(1)</statement> returns a &n; &times; &n;
+      real-valued array of all the middle vector components.  Assigning
+      to the component array will also modify the middle components of
+      the vectors in <varname>a</varname>.</para>
+ 
+      <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	    orient="port" pgwide="0">
+       <title>&array; Component Access</title>
+ 
+       <tgroup cols="2" align="left">
+        <thead>
+ 	<row>
+ 	 <entry>Function</entry>
+ 	 <entry>Effect</entry>
+ 	</row>
+        </thead>
+        <tbody>
+ 	<row>
+ 	 <entry>
+           <methodsynopsis>
+ 	   <type>UNKNOWN compute this</type> <methodname>comp</methodname>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type>int&</type>
+ 	    <parameter>i1</parameter>
+ 	   </methodparam>
+ 	  </methodsynopsis>
+          </entry>
+ 	 <entry><para>unknown: See line 1989.</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>ADD ALL <methodname>comp</methodname>s.</entry>
+ 	</row>
+        </tbody>
+       </tgroup>
+      </table>
+     </section>
+ 
+     <section id="arrays_ref-array-accessors">
+      <title>Accessors</title>
+ 
+      <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	    orient="port" pgwide="0">
+       <title>&array; Accessor Methods</title>
+ 
+       <tgroup cols="2" align="left">
+        <thead>
+ 	<row>
+ 	 <entry>Function</entry>
+ 	 <entry>Effect</entry>
+ 	</row>
+        </thead>
+        <tbody>
+ 	<row>
+ 	 <entry>
+           <methodsynopsis>
+ 	   <type>int</type> <methodname>first</methodname>
+ 	   <methodparam>
+ 	    <type>int</type>
+ 	    <parameter>d</parameter>
+ 	   </methodparam>
+ 	  </methodsynopsis>
+          </entry>
+ 	 <entry><para>unknown: See line 2050</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>ADD ALL other accessor methods, including
+ 	 <methodname>engine</methodname>.</entry>
+ 	</row>
+        </tbody>
+       </tgroup>
+      </table>
+     </section>
+ 
+ 
+     <section id="arrays_ref-array-copying">
+      <title>Copying &array;s</title>
+ 
+      <para>Explain how copied arrays and views of arrays share the
+      same underlying engine so changing values in one also affects the
+      other.  This is called a shallow copy.</para>
+     </section>
+ 
+ 
+     <section id="arrays_ref-array-utilities">
+      <title>Utility Methods</title>
+ 
+      <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	    orient="port" pgwide="0">
+       <title>&array; Utility Methods</title>
+ 
+       <tgroup cols="2" align="left">
+        <thead>
+ 	<row>
+ 	 <entry>Function</entry>
+ 	 <entry>Effect</entry>
+ 	</row>
+        </thead>
+        <tbody>
+ 	<row>
+ 	 <entry>
+           <methodsynopsis>
+ 	   <type>void</type> <methodname>makeOwnCopy</methodname>
+ 	   <void></void>
+ 	  </methodsynopsis>
+          </entry>
+ 	 <entry><para>unknown: See line 2044</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>ADD ALL other utility methods.</entry>
+ 	</row>
+        </tbody>
+       </tgroup>
+      </table>
+     </section>
+ 
+     
+     <section id="arrays_ref-array-implementation">
+      <title>Implementation Details</title>
+ 
+      <para>As a container, an &array;'s implementation is quite
+      simple.  Its <keywordname>private</keywordname>data consists of
+      an engine, and it has no <keywordname>private</keywordname>
+      functions.</para>
+ 
+      <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	    orient="port" pgwide="0">
+       <title>&array; Implementation Data</title>
+ 
+       <tgroup cols="2" align="left">
+        <thead>
+ 	<row>
+ 	 <entry>Data Member</entry>
+ 	 <entry>Meaning</entry>
+ 	</row>
+        </thead>
+        <tbody>
+ 	<row>
+ 	 <entry>
+           <fieldsynopsis>
+ 	   <modifier>private</modifier>
+ 	   <type>Engine_t</type> <varname>engine_m</varname>
+ 	  </fieldsynopsis>
+          </entry>
+ 	 <entry><para>engine computing the array's values</para></entry>
+ 	</row>
+        </tbody>
+       </tgroup>
+      </table>
+ 
+     </section>
+    </section>
+ 
+ 
+    <section id="arrays_ref-dynamicarray">
+     <title>&dynamicarray;s: Dynamically-Sized Domains</title>
+ 
+     <para>A DynamicArray is a read-write array with extra
+     create/destroy methods.  It can act just like a regular Array, but
+     can have a dynamically-changing domain.  See <filename
+ 							   class="headerfile">src/DynamicArray/DynamicArray.h</filename>.</para>
+ 
+     <para>ADD: Briefly describe what the class does and an example of
+     where it is used.</para>
+ 
+     <para>ADD: Check that its interface is actually the same as for
+     &array;.</para>
+ 
+     <para>ADD: Check that the operations on dynamic arrays are
+     actually the same as for &array;.  See <filename
+ 						     class="headerfile">src/DynamicArray/DynamicArrayOperators.h</filename>,
+     <filename
+ 	      class="headerfile">src/DynamicArray/PoomaDynamicArrayOperators.h</filename>,
+     and <filename
+ 		  class="headerfile">src/DynamicArray/VectorDynamicArrayOperators.h</filename>.</para>
+ 
+     
+     <section id="arrays_ref-dynamicarray-implementation">
+      <title>Implementation Details</title>
+ 
+      <para><type>DynamicArray</type> has no
+      <keywordname>protected</keywordname> or
+      <keywordname>private</keywordname> members.</para>
+     </section>
+    </section>
+ 
+ 
+    <section id="arrays_ref-views">
+     <title>Views of &array;s</title>
+ 
+     <para>UNFINISHED</para>
+    </section>
+ 
+ 
+    <section id="arrays_ref-assignment">
+     <title>&array; Assignments</title>
+ 
+     <para>&pooma; supports assignments to &array;s of other &array;s
+     and scalar values.  QUESTION: Is the following correct?  For the
+     former, the right-hand side array's domain must be at least as
+     large as the left-hand side array's domain.  Corresponding values
+     are copied.  Assigning a scalar value to an array ensures all the
+     array elements have the same scalar value.</para>
+ 
+     <para>UNFINISHED: Add a table containing assignment operators
+     found one lines 2097&ndash;2202.</para>
+    </section>
+ 
+ 
+    <section id="arrays_ref-print">
+     <title>Printing &array;s</title>
+ 
+     <para>&array;s support output to but not input from IO streams.
+     In particular, output to <type>ostream</type>s and file streams is
+     supported.</para>
+ 
+     <para>Add a table, using <filename
+     class="headerfile">src/Array/Array.h</filename>, lines
+     2408&ndash;2421.  See the implementation in <filename
+     class="headerfile">src/Array/PrintArray.h</filename>.</para>
+ 
+     <para>QUESTION: How does one print a &dynamicarray;.</para>
+    </section>
+ 
+ 
+    <section>
+     <title>Expressions Involving &array;s</title>
+ 
+     <para>In &pooma;, expressions may contain entire &array;s.  That
+     is, &array;s are first-class objects with respect to expressions.
+     For example, given &array;s <varname>a</varname> and
+     <varname>b</varname>, the expression <statement>a + b</statement>
+     is equivalent to an array containing the element-wise sum of the
+     two arrays.</para>
+ 
+     <para>Any finite number of the operators listed below can be used
+     in an expression.  The precedence and order of operation is the
+     same as with ordinary built-in types.</para>
+ 
+     <para>QUESTION: Do &field;s also support the same set of
+     operations?</para>
+ 
+     <para>QUESTION: Some operations in <filename
+ 						 class="headerfile">src/Field/FieldOperators.h</filename> use both
+     &array; and &field;.  Do we list them here or in the &field;
+     section or both or somewhere else?</para>
+ 
+     <para>In the table below, &array; supplants the exact return types
+     because they are complicated and rarely need to be explicitly
+     written down.</para>
+ 
+     <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	   orient="port" pgwide="0">
+      <title>Operators on &array;</title>
+ 
+      <tgroup cols="2" align="left">
+       <thead>
+        <row>
+ 	<entry>Operator</entry>
+ 	<entry>Value</entry>
+        </row>
+       </thead>
+       <tbody>
+        <row>
+ 	<entry>
+           <funcsynopsis>
+ 	  <funcprototype>
+ 	   <funcdef>Array <function>acos</function></funcdef>
+ 	   <paramdef>const <parameter>Array&lt;Dim,T,EngineTag&gt;& a</parameter></paramdef>
+ 	  </funcprototype>
+ 	 </funcsynopsis>
+          </entry>
+ 	<entry><para>an array containing the element-wise inverse
+          cosine of the array <parameter>a</parameter></para></entry>
+        </row>
+        <row>
+ 	<entry>ADD ALL other operators appearing in <filename
+ 							      class="headerfile">src/Array/ArrayOperators.h</filename>,
+         <filename
+ 		  class="headerfile">src/Array/ArrayOperatorSpecializations.h</filename>,
+         <filename
+ 		  class="headerfile">src/Array/PoomaArrayOperators.h</filename>,
+         and <filename
+ 		      class="headerfile">src/Array/VectorArrayOperators.h</filename>.</entry>
+        </row>
+       </tbody>
+      </tgroup>
+     </table>
+ 
+     <para>FINISH: Write one or two examples or refer to ones
+     previously in the text.</para>
+    </section>
+ 
+ 
+    <section id="arrays_ref-reductions">
+     <title>Reducing All &array; Elements to One Value</title>
+ 
+     <para>These reduction functions repeatedly apply a binary
+     operation to all array elements to yield a value.  These functions
+     are similar to the Standard Template Library's
+     <function>accumulate</function> function.  For example,
+     <function>sum</function> repeatedly applies the binary plus
+     operator to all array elements, yielding the sum of all array
+     elements.</para>
+ 
+     <para>FINISH: What order of operation, if any, is
+     guaranteed?</para>
+ 
+     <para>FINISH: Add a table of the functions in <filename
+ 							    class="headerfile">src/Array/Reductions.h</filename>.</para>
+ 
+     <para>How does one use one's own binary function?  See <filename
+     class="headerfile">src/Engine/Reduction.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="arrays_ref-utilities">
+     <title>Utility Functions</title>
+ 
+     <section id="arrays_ref-utilities-compression">
+      <title>Compressed Data</title>
+ 
+      <para>Add a table containing
+      <function>elementsCompressed</function>,
+      <function>compressed</function>, <function>compress</function>,
+      and <function>uncompress</function>.</para>
+     </section>
+ 
+ 
+     <section id="arrays_ref-utilities-sizes">
+      <title>Centering Sizes and Number of Materials</title>
+ 
+      <para>ADD: a description of <function>numMaterials</function> and
+      <function>centeringSize</function> found in <filename
+      class="headerfile">src/Field/Field.h</filename>.  These functions
+      are meaningless for &array; but are provided for consistency with
+      &field;.</para>
+     </section>
+ 
+     <section id="arrays_ref-utilities-subfield">
+      <title>Obtaining Subfields</title>
+ 
+      <para>ADD: a description of <function>subField</function> found
+      in <filename class="headerfile">src/Field/Field.h</filename>.
+      This function, meaningless for &array;, is provided for
+      consistency with &field;.</para>
+     </section>
+    </section>
+ 
+ 
+    <section id="arrays_ref-tmp">
+     <title>TMP: What do we do with these &hellip;? Remove this
+     section.</title>
+ 
+     <para>QUESTION: Do we describe the &leaffunctor;s specialized for
+     &array;s in <filename
+     class="headerfile">src/Array/Array.h</filename> or in the &pete;
+     reference section?  What about the functions in <filename
+     class="headerfile">src/Array/CreateLeaf.h</filename>?</para>
+ 
+     <para>QUESTION: What is an <type>EngineFunctor</type>?  We
+     probably should describe it in an analogous way as for
+     &leaffunctor;s.</para>
+ 
+     <para>QUESTION: Where do we write about
+     <type>ExpressionTraits</type> for &array;s?</para>
+ 
+     <para>QUESTION: Do we describe the <type>ElementProperties</type>
+     specialization at this place or in its section?</para>
+ 
+     <para>QUESTION: Do we describe the <type>Patch</type>
+     specialization for &array;s (<filename
+     class="headerfile">src/Array/Array.h</filename>:1300) in this
+     place or in a section for patches?</para>
+    </section>
+   </chapter>
+ 
+ 
+   <chapter id="fields_ref">
+    <title>&field;s</title>
+ 
+    <para>An &array; is a set of values indexed by
+    coordinates, one value per coordinate.  It models the computer
+    science idea of an array.  Similarly, a &field; is a set of values
+    indexed by coordinate.  It models the mathematical and physical
+    idea of a field represented by a grid of rectangular cells, each
+    having at least one value.  A &field;'s functionality is a superset
+    of an &array;'s functionality because:
+     <itemizedlist>
+      <listitem>
+       <para>A &field; is distributed through space so one can compute
+       the distances between cells.</para>
+      </listitem>
+      <listitem>
+       <para>Each cell can hold multiple values.  For example, a
+       rectangular cell can have one value on each of its faces.</para>
+      </listitem>
+      <listitem>
+       <para>Multiple materials can share the same cell.  For example,
+       different values can be stored in the same cell for carbon,
+       oxygen, and nitrogen.</para>
+      </listitem>
+     </itemizedlist>
+    Also, &field;s' values can be related by relations.  Thus, if one
+    field's values change, a dependent field's values can be
+    automatically computed when needed.  FIXME: See also the unfinished
+    works chapter's entry concerning relations and arrays.</para>
+ 
+    <para>QUESTION: Should we add a picture comparing and contrasting
+    an array and a field?</para>
+ 
+    <para>QUESTION: How much structure can be copied from the &array;
+    chapter?</para>
+ 
+    <para>QUESTION: Where is <type>NewMeshTag</type>, defined in
+    <filename class="headerfile">src/Field/Field.h</filename>,
+    used?</para>
+ 
+    <para>QUESTION: Do we describe the &leaffunctor;s specialized for
+    &field;s in <filename
+    class="headerfile">src/Field/Field.h</filename> or in the &pete;
+    reference section?  Use the same decision for &array;s.</para>
+ 
+    <para>QUESTION: What do the structure and functions in <filename
+    class="headerfile">src/Field/Mesh/PositionFunctions.h</filename>
+    do?</para>
+ 
+ 
+    <section id="fields_ref-field">
+     <title>The &field; Container</title>
+ 
+     <para>ADD: table of template parameters and table of compile-time
+     types and values.</para>
+ 
+ 
+     <section id="fields_ref-field-constructors">
+      <title>Constructors and Destructors</title>
+ 
+      <para>ADD: this section similar to &array;s's constructor and
+      destructor section.</para>
+     </section>
+ 
+     <section id="fields_ref-field-initializers">
+      <title>Initializers</title>
+ 
+      <para>Add a table.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-access">
+      <title>Element Access</title>
+ 
+      <para>ADD: a table ala &array;.  Be sure to include
+      <methodname>all</methodname>.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-component">
+      <title>Component Access</title>
+ 
+      <para>ADD: a table ala &array;.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-subfields">
+      <title>Obtaining Subfields</title>
+ 
+      <para>ADD: discussion and a table listing ways to obtain
+       subfields.  Although the implementation may treat subfield views
+       and other field views similarly (?Is this true?), they are
+       conceptually different ideas so we present them
+       separately.</para>
+ 
+      <para>See <filename
+ 			 class="headerfile">src/Field/Field.h</filename>'s
+       <methodname>operator[]</methodname>,
+       <methodname>subField</methodname>, &hellip;,
+       <methodname>material</methodname>.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-relations">
+      <title>Supporting Relations</title>
+ 
+      <para>ADD: a table with the member functions including
+      <methodname>addRelation</methodname>,
+      <methodname>removeRelations</methodname>,
+      <methodname>applyRelations</methodname>, and
+      <methodname>setDirty</methodname>.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-accessors">
+      <title>Accessors</title>
+ 
+      <para>ADD: a table using lines like <filename
+      class="headerfile">src/Field/Field.h</filename>:1243&ndash;1333.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-utilities">
+      <title>Utility Methods</title>
+ 
+      <para>ADD: a table including
+       <methodname>makeOwnCopy</methodname>.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-implementation">
+      <title>Implementation Details</title>
+ 
+      <para>ADD: a table similar to &array;'s.</para>
+ 
+     </section>
+ 
+    </section>
+ 
+ 
+    <section id="engines_ref-views">
+     <title>Views of &field;s</title>
+ 
+     <para>Be sure to relate to &array; views.  Note only three
+     dimensions are supported.</para>
+ 
+     <para>Be sure to describe <statement>f[i]</statement>.  Does this
+     refer to a particular material or a particular value within a
+     cell?  I do not remember.  See <type>SubFieldView</type> in
+     <filename class="headerfile">src/Field/Field.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-assignment">
+     <title>&field; Assignments</title>
+ 
+     <para>ADD: Describe supported assignments, relating to &array;'s
+     assignments.</para>
+ 
+     <para>UNFINISHED: Add a table containing assignment operators
+     found on <filename
+ 		       class="headerfile">src/Field/Field.h</filename>:2097&ndash;2202
+     and 1512&ndash;1611.</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-print">
+     <title>Printing &field;s</title>
+ 
+     <para>QUESTION: How similar is this to printing &array;s?</para>
+ 
+     <para>&field;s support output to but not input from IO streams.
+     In particular, output to <type>ostream</type>s and file streams is
+     supported.</para>
+ 
+     <para>Add a table, using <filename
+ 				       class="headerfile">src/Field/Field.h</filename>, lines
+     1996&ndash;2009.  See the implementation in <filename
+ 							  class="headerfile">src/Field/PrintField.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-reductions">
+     <title>Combining &field; Elements</title>
+ 
+     <para>Like &array;s, &field;s support reduction of all elements to
+     one value.  Additionally, the latter supports computing a field's
+     values using field stencils.  QUESTION: How do I describe this
+     with a minimum of jargon?</para>
+ 
+     <para>ADD: something similar to &array; reductions.</para>
+ 
+     <para>FINISH: Add a table of the functions in <filename
+     class="headerfile">src/Field/FieldReductions.h</filename>.</para>
+ 
+     <para>FINISH: Add a table of the functions in <filename
+     class="headerfile">src/Field/DiffOps/FieldOffsetReductions.h</filename>.
+     QUESTION: Why is only <function>sum</function> defined?</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-expressions">
+     <title>Expressions Involving &field;s</title>
+ 
+     <para>Do something similar to &array;'s section.  See the
+     operations defined in <filename
+     class="headerfile">src/Field/FieldOperators.h</filename>,
+     <filename
+     class="headerfile">src/Field/FieldOperatorSpecializations.h</filename>,
+     <filename
+     class="headerfile">src/Field/PoomaFieldOperators.h</filename>, and
+     <filename
+     class="headerfile">src/Field/VectorFieldOperators.h</filename>.</para>
+ 
+     <para>Some operations involve both &array; and &field;
+     parameters.  Where do we list them?</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-stencils">
+     <title>&field; Stencils: Faster, Local Computations</title>
+ 
+     <para>ADD: a description of a stencil.  Why is it needed?  How
+     does a user use it?  How does a user know when to use one?  Add
+     documentation of the material from <filename
+     class="headerfile">src/Field/DiffOps/FieldStencil.h</filename>.</para>
+ 
+     <para>How is <type>FieldShiftEngine</type> used by &field;
+     stencils?  Should it be described here or in the &engine; section?
+     See the the code in <filename
+     class="headerfile">src/Field/DiffOps/FieldShiftEngine.h</filename>.</para>
+    </section>
+ 
+ 
+    <section>
+     <title>Cell Volumes, Face Areas, Edge Lengths, Normals</title>
+ 
+     <para>ADD: a description of these functions.  See <filename
+     class="headerfile">src/Field/Mesh/MeshFunctions.h</filename>.
+     These are initialized in, e.g., <filename
+     class="headerfile">src/Field/Mesh/UniformRectilinearMesh.h</filename>.
+     Note that these do not work for <type>NoMesh</type>.</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-div">
+     <title>Divergence Operators</title>
+ 
+     <para>ADD: a table having divergence operators, explaining the
+     current restrictions imposed by what is implemented.  See
+     <filename class="headerfile">src/Field/DiffOps/Div.h</filename>
+     and <filename
+     class="headerfile">src/Field/DiffOps/Div.UR.h</filename>.  What
+     restrictions does <quote>UR</quote> (mesh) connote?</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-utilities">
+     <title>Utility Functions</title>
+ 
+     <section id="fields_ref-utilities-compression">
+      <title>Compressed Data</title>
+ 
+      <para>Add a table containing
+      <function>elementsCompressed</function>,
+      <function>compressed</function>, <function>compress</function>,
+      and <function>uncompress</function>.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-utilities-sizes">
+      <title>Centering Sizes and Number of Materials</title>
+ 
+      <para>ADD: a description of <function>numMaterials</function> and
+      <function>centeringSize</function> found in <filename
+      class="headerfile">src/Field/Field.h</filename>.</para>
+ 
+      <para>QUESTION: How do these relate to any method functions?</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-utilities-subfield">
+      <title>Obtaining Subfields</title>
+ 
+      <para>ADD: a description of <function>subField</function> found
+      in <filename
+      class="headerfile">src/Field/Field.h</filename>.</para>
+     </section>
+ 
+    </section>
+ 
+ 
+    <section id="fields_ref-centerings">
+     <title>&field; Centerings</title>
+ 
+     <para>DO: Describe the purpose of a centering and its definition.
+     Describe the ability to obtain canonical centerings.  Explain how
+     to construct a unique centering.  See <filename
+     class="headerfile">src/Field/FieldDentering.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-fieldoffsets">
+     <title>Relative &field; Positions</title>
+ 
+     <para>Permit specifying field positions relative to a field
+     location.  Describe <type>FieldOffset</type> and
+     <type>FieldOffsetList</type>.  See <filename
+     class="headerfile">src/Field/FieldOffset.h</filename></para>
+    </section>
+ 
+ 
+    <section id="fields_ref-nearestneighbors">
+     <title>Computing Close-by Field Positions</title>
+ 
+     <para>Given a field location, return the set of field locations
+     that are closest using ?Manhattan? distance.  See <filename
+     class="headerfile">src/Field/NearestNeighbors.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-mesh">
+     <title>Mesh ???</title>
+ 
+     <para>Unlike &array;s, &field;s are distributed throughout space
+     so distances between values within the &field can be computed.  A
+     &field;'s mesh stores this spatial distribution.</para>
+ 
+     <para>QUESTION: What do we need to write about meshes?  What is
+     unimportant implementation and what should be described in this
+     reference section?</para>
+ 
+     <para>QUESTION: Where in here should emphasize vertex, not cell,
+     positions?  <quote>VERTEX</quote> appears repeatedly in <filename
+     class="headerfile">src/Field/Mesh/NoMesh.h</filename>.</para>
+ 
+     <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	   orient="port" pgwide="0">
+      <title>Mesh Types</title>
+ 
+      <tgroup cols="2" align="left">
+       <thead>
+        <row>
+ 	<entry>Mesh Type</entry>
+ 	<entry>Description</entry>
+        </row>
+       </thead>
+       <tbody>
+        <row>
+ 	<entry><type>NoMesh&lt;Dim&gt;</type></entry>
+ 	<entry><para>no physical spacing, causing a &field; to mimic
+ 	an &array; with multiple engines.</para></entry>
+        </row>
+        <row>
+ 	<entry><type>UniformRectilinearMesh&lt;Dim,T&gt;</type></entry>
+ 	<entry><para>physical spacing formed by the Cartesian product
+ 	of ????.</para></entry>
+        </row>
+       </tbody>
+      </tgroup>
+     </table>
+ 
+ 
+     <section id="fields_ref-mesh-accessors">
+      <title>Mesh Accessors</title>
+ 
+      <para>ADD: a table listing accessors, explaining the difference
+      between (physical and total) and (cell and vertex) domains.  See
+      <filename class="headerfile">src/Field/Mesh/NoMesh.h</filename>.
+      Also, include <function>spacings</function> and
+      <function>origin</function> in <filename
+      class="headerfile">src/Field/Mesh/UniformRectilinearMesh.h</filename>.
+      Note <type>NoMesh</type> does not provide the latter two.</para>
+     </section>
+ 
+    </section>
+ 
+ 
+    <section id="fields_ref-tmp">
+     <title>TMP: What do we do with these &hellip;? Remove this
+     section.</title>
+ 
+     <para>QUESTION: Do we describe the <type>Patch</type>
+     specialization for &field; at this place or in some common place?
+     Follow &array;'s lead.</para>
+ 
+     <para>QUESTION: Where do we describe <type>CreateLeaf</type> and
+     <type>MakeFieldReturn</type> in <filename
+     class="headerfile">src/Field/FieldCreateLeaf.h</filename> and
+     <filename
+     class="headerfile">src/Field/FieldMakeReturn.h</filename>.</para>
+ 
+     <para>QUESTION: What do we do with <type>FieldEnginePatch</type>
+     in <filename
+     class="headerfile">src/Field/FieldEngine/FieldEnginePatch.h</filename>.</para>
+    </section>
+   </chapter>
+ 
+ 
+   <chapter id="engines_ref">
+    <title>&engine;s</title>
+ 
+    <para>From a user's point of view, a container makes data available
+    for reading and writing.  In fact, the container's &engine; stores
+    the data or, if the data is computed, performs a computation to
+    yield the data.</para>
+ 
+    <para>FINISH: Introduce the various types of engines.  Add a table
+    with a short description of each engine type.</para>
+ 
+    <para>FINISH: First, we specify a generic &engine;'s interface.
+    Then, we present &engine; specializations.</para>
+    
+    <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	  orient="port" pgwide="0">
+     <title>Types of &engine;s</title>
+ 
+     <tgroup cols="3" align="left">
+      <thead>
+       <row>
+        <entry>Engine Type</entry>
+        <entry>Engine Tag</entry>
+        <entry>Description</entry>
+       </row>
+      </thead>
+      <tbody>
+       <row>
+        <entry><para>Brick</para></entry>
+        <entry><para><literal>Brick</literal></para></entry>
+        <entry><para>Explicitly store all elements in, e.g., a &cc;
+ 	array.</para></entry>
+       </row>
+       <row>
+        <entry><para>Compressible</para></entry>
+        <entry><para><literal>CompressibleBrick</literal></para></entry>
+        <entry><para>If all values are the same, use constant storage
+ 	for that single value.  Otherwise, explicitly store all
+ 	elements.</para></entry>
+       </row>
+       <row>
+        <entry><para>Constant</para></entry>
+        <entry><para><literal>ConstantFunction</literal></para></entry>
+        <entry><para>Returns the same constant value for all
+        indices.</para></entry>
+       </row>
+       <row>
+        <entry><para>Dynamic</para></entry>
+        <entry><para><literal>Dynamic</literal></para></entry>
+        <entry><para>Manages a contiguous, local, one-dimensional,
+        dynamically resizable block of data.</para></entry>
+       </row>
+       <row>
+        <entry><para>Component Forwarding</para></entry>
+        <entry><para><literal>CompFwd&lt;EngineTag,
+        Components&gt;</literal></para></entry>
+        <entry><para>Returns the specified components from
+        <literal>EngineTag</literal>'s engine.  Components are
+        <quote>pieces</quote> of multi-value elements such as vectors
+        and tensors.</para></entry>
+       </row>
+       <row>
+        <entry><para>Expression</para></entry>
+        <entry><para><literal>ExpressionTag&lt;Expr&gt;</literal></para></entry>
+        <entry><para>Returns the value of the specified &pete;
+        expression.</para></entry>
+       </row>
+       <row>
+        <entry><para>Index Function</para></entry>
+        <entry><para><literal>IndexFunction&lt;Functor&gt;</literal></para></entry>
+        <entry><para>Makes the function
+        <literal>Functor</literal>accepting indices mimic an
+        array.</para></entry>
+       </row>
+       <row>
+        <entry><para>MultiPatch</para></entry>
+        <entry><para><literal>MultiPatch&lt;LayoutTag,PatchTag&gt;</literal></para></entry>
+        <entry><para>Support distributed computation using several
+        processors (???contexts???).  <literal>LayoutTag</literal>
+        indicates how the entire array is distributed among the
+        processors.  Each processor uses a <literal>PatchTag</literal>
+        engine.</para></entry>
+       </row>
+       <row>
+        <entry><para>Remote</para></entry>
+        <entry><para><literal>Remote&lt;EngineTag&gt;</literal></para></entry>
+        <entry><para>unknown</para></entry>
+       </row>
+       <row>
+        <entry><para>Remote Dynamic</para></entry>
+        <entry><para><literal>Remote&lt;Dynamic&gt;</literal></para></entry>
+        <entry><para>unknown: specialization</para></entry>
+       </row>
+       <row>
+        <entry><para>Stencil</para></entry>
+        <entry><para><literal>StencilEngine&lt;Function,
+        Expression&gt;</literal></para></entry>
+        <entry><para>Returns values computed by applying the
+        user-specified function to sets of contiguous values in the
+        given engine or container.  Compare with user function
+        engines.</para></entry>
+       </row>
+       <row>
+        <entry><para>User Function</para></entry>
+        <entry><para><literal>UserFunctionEngine&lt;UserFunction,Expression&gt;</literal></para></entry>
+        <entry><para> Returns values computed by applying the
+        user-specified function to the given engine or container.
+        QUESTION: Is the following claim correct? For each returned
+        value, only one value from the engine or container is
+        used.</para></entry>
+       </row>
+      </tbody>
+     </tgroup>
+    </table>
+ 
+    <para>QUESTION: Where do we describe views?</para>
+ 
+    <para>QUESTION: What does <type>NewEngine</type> do?  Should it be
+     described when describing views?  Should it be omitted as an
+     implementation detail?</para>
+ 
+    <para>QUESTION: Where do we describe &engine; patches found in
+    <filename class="headerfile">src/Engine/EnginePatch.h</filename>?
+    All patch data in a separate chapter or engine-specific pieces in
+    this chapter?</para>
+ 
+    <para>QUESTION: What is <function>notifyEngineWrite</function>?
+    See also <filename
+ 		      class="headerfile">src/Engine/NotifyEngineWrite.h</filename>.</para>
+ 
+    <para>QUESTION: What aspect of MultiPatch uses <type>IsValid</type> in
+    <filename
+ 	     class="headerfile">src/Engine/IsValidLocation.h</filename>?</para>
+ 
+    <para>QUESTION: Who uses intersections?  Where should this be
+    described?  See <filename
+ 			     class="headerfile">src/Engine/Intersector.h</filename>, <filename
+ 											       class="headerfile">src/Engine/IntersectEngine.h</filename>, and
+    <filename
+ 	     class="headerfile">src/Engine/ViewEngine.h</filename>.</para>
+ 
+    <section id="engines_ref-compile_time_interface">
+     <title>&engine; Compile-Time Interface</title>
+ 
+     <para>ADD: a table of template parameters ala &array;.  ADD:
+     compile-time types and values.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-constructors">
+     <title>Constructors and Destructors</title>
+ 
+     <para>ADD: a table of constructors and destructors ala
+     &array;'s.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-access">
+     <title>Element Access</title>
+ 
+     <para>ADD: a table with <methodname>read</methodname> and
+     <methodname>operator()</methodname>.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-accessors">
+     <title>Accessors</title>
+ 
+     <para>ADD: a table of accessors.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-assignments">
+     <title>&engine; Assignments</title>
+ 
+     <para>similar to &array;'s assignments.  shallow copies.  ADD: a
+     table with one entry</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-utilities">
+     <title>Utility Methods</title>
+ 
+     <para>ADD: a table including
+     <methodname>makeOwnCopy</methodname>.</para>
+ 
+     <para>QUESTION: What are <methodname>dataObject</methodname>,
+     <methodname>isShared</methodname>, and related methods?</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-implementation">
+     <title>Implementation Details</title>
+ 
+     <para>ADD: this section.  Explain that
+     <varname>dataBlock_m</varname> and <varname>data_m</varname> point
+     to the same place.  The latter speeds access, but what is the
+     purpose of the former?</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-brick">
+     <title>Brick and BrickView Engines</title>
+ 
+     <para>ADD: description of what a brick means.  ADD: whatever
+     specializations the class has, e.g.,
+     <methodname>offset</methodname>.</para>
+ 
+     <para>QUESTION: What does <type>DoubleSliceHelper</type> do?</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-compressible">
+     <title>Compressible Brick and BrickView Engines</title>
+ 
+     <para>ADD this.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-dynamic">
+     <title>Dynamic and DynamicView Engines:</title>
+ 
+     <para>ADD this.  Manages a contiguous, local, resizable, 1D block
+     of data.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-components">
+     <title>Component Engines</title>
+ 
+     <para>I believe these implement array component-forwarding.  See
+     <filename
+ 	      class="headerfile">src/Engine/ForwardingEngine.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-expressions">
+     <title>Expression Engines</title>
+ 
+     <para>Should this be described in the &pete; section?  Unlikely.
+     See <filename
+ 		  class="headerfile">src/Engine/ExpressionEngine.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-enginefunctor">
+     <title>&engine; Functors</title>
+ 
+     <para>QUESTION: What is an <type>EngineFunctor</type>?  Should it
+     have its own section?  See <filename
+ 					 class="headerfile">src/Engine/EngineFunctor.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-fieldengines">
+     <title><type>FieldEngine</type>: A Hierarchy of &engine;s</title>
+ 
+     <para>A &field; consists of a hierarchy of materials and
+     centerings.  These are implemented using a hierarchy of engines.
+     See <filename
+ 		  class="headerfile">src/Field/FieldEngine/FieldEngine.h</filename>
+     and <filename
+ 		  class="headerfile">src/Field/FieldEngine/FieldEngine.ExprEngine.h</filename>.</para>
+    </section>
+   </chapter>
+ 
+ 
+   <chapter id="benchmarks_ref">
+    <title>&benchmark; Programs</title>
+ 
+    <para>Explain how to use &benchmark; programs, especially the
+    options.  Explain how to write a &benchmark; program.  See also
+    <filename class="headerfile">src/Utilities/Benchmark.h</filename>
+    and <filename
+    class="libraryfile">src/Utilities/Benchmark.cmpl.cpp</filename>.</para>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="layouts_ref">
+    <title>Layouts and Partitions: Distribute Computation Among
+    Contexts</title>
+ 
+    <para>QUESTION: What is the difference between
+    <type>ReplicatedTag</type> and <type>DistributedTag</type>?</para>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="pete_ref">
+    <title>&pete;: Evaluating Parallel Expressions</title>
+ 
+    <section>
+     <title>UNKNOWN</title>
+ 
+     <section id="pete_ref-unknown-leaf-tags">
+      <title>Leaf Tag Classes</title>
+ 
+      <para><type>NotifyPreReadTag</type> indicates a term is about to
+      be read.  Why is this needed?  Defined in <filename
+      class="headerfile">src/Utilities/NotifyPreRead.h</filename>.</para>
+     </section>
+    </section>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="views_ref">
+    <title>Views</title>
+ 
+    <para>QUESTION: Should this have its own chapter or be part of a
+    container chapter?</para>
+ 
+    <para>Describe <type>View0</type>, <type>View1</type>, &hellip;,
+    <type>View7</type> and <type>View1Implementation</type>.</para>
+ 
+    <para>QUESTION: What causes the need for <type>AltView0</type> and
+    <type>AltComponentView</type>?</para>
+ 
+    <para>Be sure to describe <type>ComponentView</type> in the same
+    place.  This is specialized for &array;s in <filename
+    class="headerfile">src/Array/Array.h</filename>:1323&ndash;1382.</para>
+ 
+    <section>
+     <title><type>ViewIndexer&lt;Dim,Dim2&gt;</type></title>
+ 
+     <para>Defined in <filename
+     class="headerfile">src/Utilities/ViewIndexer.h</filename>, this
+     type translates indices between a domain and a view of it.</para>
+    </section>
+   </chapter>
+ 
+   <chapter id="threads_ref">
+    <title>Threads</title>
+ 
+    <para>Perhaps include information in <filename
+    class="headerfile">src/Engine/DataObject.h</filename>.</para>
+ 
+    <para>&pooma; options include UNFINISHED</para>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="utilities_ref">
+    <title>Utility Types</title>
+ 
+    <para>TMP: What is a good order?</para>
+ 
+    <section id="utilities_ref-options">
+     <title><type>Options</type>: Varying Run-Time Execution</title>
+ 
+     <para>Each &pooma; executable has a <type>Options</type> object,
+     created by <function>Pooma::initialize</function>, storing
+     run-time configurable values found in <varname>argv</varname>.
+     Default options are found in
+     <methodname>Options::usage</methodname>.</para>
+ 
+     <para>See <filename
+     class="headerfile">src/Utilities/Options.h</filename> and
+     <filename
+     class="libraryfile">src/Utilities/Options.cmpl.cpp</filename>.</para>
+ 
+     <para>Scatter the specific options to other parts of the
+     manual.</para>
+    </section>
+ 
+    <section id="utilities_ref-assertions">
+     <title>Check Correctness: <type>CTAssert</type>,
+     <type>PAssert</type>, <type>PInsist</type>,
+     <type>SameType</type></title>
+ 
+     <para>Assertions ensure program invariants are obeyed.
+     <type>CTAssert</type>, checked at compile time, incur no run-time
+     cost.  <type>PAssert</type> and <type>PInsist</type> are checked
+     to run-time, the latter producing an explanatory message if the
+     assertion fails.  Compiling with <envar>NOCTAssert</envar> and
+     <envar>NOPTAssert</envar> disable these checks.  Compiling with just
+     <envar>NOPTAssert</envar> disables only the run-time checks.</para>
+ 
+     <para><type>SameType</type> ensures, at compile-time, two types
+     are the same.</para>
+ 
+     <para>These are implemented in <filename
+     class="headerfile">src/Utilities/PAssert.h</filename> and
+     <filename
+     class="libraryfile">src/Utilities/PAssert.cmpl.cpp</filename>.</para>
+    </section>
+ 
+    <section id="utilities_ref-clock">
+     <title><type>Clock</type>: Measuring a Program's Execution Time</title>
+ 
+     <para>See <filename
+     class="headerfile">src/Utilities/Clock.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="utilities_ref-smart_pointers">
+     <title>Smart Pointers: <type>RefCountedPtr</type>,
+     <type>RefCountedBlockPtr</type>, and
+     <type>DataBlockPtr</type></title>
+ 
+     <para>See <filename
+     class="headerfile">src/Utilities/{RefCountedPtr,RefCountedBlockPtr,DataBlockPtr}.h</filename>.
+     <filename class="headerfile">src/Utilities/RefCounted.h</filename>
+     helps implement it.  <type>DataBlockPtr</type> uses
+     &smarts;.</para>
+    </section>
+ 
+    <section id="utilities_ref-inform">
+     <title><type>Inform</type>: Formatted Output for Multi-context
+     Execution</title>
+ 
+     <para>See <filename
+     class="headerfile">src/Utilities/Inform.h</filename> and <filename
+     class="libraryfile">src/Utilities/Inform.cmpl.cpp</filename>.</para>
+    </section>
+ 
+    <section id="utilities_ref-statistics">
+     <title><type>Statistics</type>: Report &pooma; Execution Statistics</title>
+ 
+     <para>Collect and print execution statistics.  Defined in
+     <filename
+     class="headerfile">src/Utilities/Statistics.h</filename>.</para>
+    </section>
+ 
+    <section id="utilities_ref-random_numbers">
+     <title>Random Numbers: <type>Unique</type></title>
+ 
+     <para>See <filename
+     class="headerfile">src/Utilities/Unique.h</filename>.</para>
+    </section>
+   </chapter>
+ 
+ 
+   <chapter id="implementation_types_ref">
+    <title>Types for Implementing &pooma;</title>
+ 
+    <para>TMP: What is a good order?</para>
+ 
+    <para>Describe types defined to implement &pooma; but that users do
+    not directly use.  This chapter has lower priority than other
+    chapters since users (hopefully) do not need to know about these
+    classes.</para>
+ 
+    <section id="implementation_types_ref-tester">
+     <title><type>Tester</type>: Check Implementation Correctness</title>
+ 
+     <para>&pooma; implementation test programs frequently consist of a
+     series of operations followed by correctness checks.  The
+     <type>Tester</type> object supports these tests, returning a
+     boolean whether all the correctness checks yield true.  Under
+     verbose output, messages are printed for each test.  See <filename
+     class="headerfile">src/Utilities/Tester.h</filename>.</para>
+    </section>
+ 
+    <section id="implementation_types_ref-elementproperties">
+     <title><type>ElementProperties&lt;T&gt;</type>: Properties a Type
+     Supports</title>
+ 
+     <para>This traits class permits optimizations in other templated
+     classes.  See <filename
+     class="headerfile">src/Utilities/ElementProperties.h</filename>.</para>
+ 
+    </section>
+ 
+    <section id="implementation_types_ref-typeinfo">
+     <title><type>TypeInfo&lt;T&gt;</type>: Print a String Describing
+     the Type</title>
+ 
+     <para>Print a string describing the type.  Defined in <filename
+     class="headerfile">src/Utilities/TypeInfo.h</filename>.  It is
+     specialized for other types in other files, e.g., <filename
+     class="headerfile">src/Engine/EngineTypeInfo.h</filename> and
+     <filename class="headerfile">src/Field/FieldTypeInfo.h</filename>.
+     Is this a compile-time version of RTTI?</para>
+    </section>
+ 
+    <section id="implementation_types_ref-looputils">
+     <title><type>LoopUtils</type>: Loop Computations at Compile Time</title>
+ 
+     <para>At compile time, <type>LoopUtils</type> supports copying
+     between arrays and computing the dot product of arrays.  See
+     <filename
+ 	      class="headerfile">src/Utilities/MetaProg.h</filename>.</para>
+    </section>
+ 
+    <section id="implementation_types_ref-modelelement">
+     <title><type>ModelElement&lt;T&gt;</type>: Wrap a Type</title>
+ 
+     <para>A wrapper class used to differentiate overloaded functions.
+     Defined in <filename
+     class="headerfile">src/Utilities/ModelElement.h</filename>.  Used
+     only by &array; and <type>DynamicArray</type>.</para>
+    </section>
+ 
+    <section id="implementation_types_ref-wrappedint">
+     <title><type>WrappedInt&lt;int&gt;</type>: Wrap a Number</title>
+ 
+     <para>A wrapper class used to differentiate overloaded functions
+     among different integers.  Defined in <filename
+     class="headerfile">src/Utilities/WrappedInt.h</filename>.  Is this
+     class deprecated?  Is it even necessary?</para>
+    </section>
+    
+    <section id="implementation_types_ref-empty_classes">
+     <title>Supporting Empty Classes</title>
+ 
+     <para>The <type>NoInit</type> tag class indicates certain
+     initializations should be skipped.  Defined in <filename
+     class="headerfile">src/Utilities/NoInit.h</filename>.</para>
+ 
+     <para>FIXME: Should be macro, not function.
+     <function>POOMA_PURIFY_CONSTRUCTORS</function> generates an empty
+     constructor, copy constructor, and destructor to avoid &purify;
+     warnings.  Defined in <filename
+     class="headerfile">src/Utilities/PurifyConstructors.h</filename>.</para>
+ 
+    </section>
+ 
+    <section id="implementation_types_ref-pool">
+     <title><type>Pooled&lt;T&gt;</type>: Fast Memory Allocation of
+     Small Blocks</title>
+ 
+     <para><type>Pooled&lt;T&gt;</type> speeds allocation and
+     deallocation of memory blocks for small objects with
+     type&nbsp;<type>T</type>.  Defined in <filename
+     class="headerfile">src/Utilities/Pooled.h</filename>, it is
+     implemented in <filename
+     class="headerfile">src/Utilities/Pool.h</filename> and <filename
+     class="libraryfile">src/Utilities/Pool.cmpl.cpp</filename>.
+     <filename class="headerfile">src/Utilities/StaticPool.h</filename>
+     no longer seems to be used.</para>
+    </section>
+ 
+    <section id="implementation_types_ref-uninitialized_vector">
+     <title><type>UninitializedVector&lt;T,Dim&gt;</type>: Create
+     Without Initializing</title>
+ 
+     <para>This class optimizes creation of an array of objects by
+     avoiding running the default constructors.  Later initialization
+     can occur, perhaps using a loop that can be unrolled.  Defined in
+     <filename
+     class="headerfile">src/Utilities/UninitializedVector.h</filename>,
+     this is used only by <type>DomainTraits</type>.</para>
+    </section>
+   </chapter>
+ 
+   <chapter id="implementation_algorithms_ref">
+    <title>Algorithms for Implementing &pooma;</title>
+ 
+    <para>In <filename>src/Utilities/algorithms.h</filename>,
+    <function>copy</function>, <function>delete_back</function>, and
+    <function>delete_shiftup</function> provide additional algorithms
+    using iterators.</para>
+   </chapter>
+ 
+ 
+   <chapter id="where-place-these_ref">
+    <title>TMP: Where do we describe these files?</title>
+ 
+    <itemizedlist>
+     <listitem>
+      <para><filename
+ 		     class="headerfile">src/Utilities/Conform.h</filename>: tag for
+      checking whether terms in expression have conforming
+      domains</para>
+     </listitem>
+ 
+     <listitem>
+      <para><filename
+ 		     class="headerfile">src/Utilities/DerefIterator.h</filename>:
+      <type>DerefIterator&lt;T&gt;</type> and
+      <type>ConstDerefIterator&lt;T&gt;</type> automatically
+      dereference themselves to maintain <literal>const</literal>
+      correctness.  <!-- FIXME: s/literal/keyword/ --></para>
+     </listitem>
+ 
+     <listitem>
+      <para><filename
+ 		     class="headerfile">src/Utilities/Observable.h</filename>,
+      <filename class="headerfile">src/Utilities/Observer.h</filename>,
+      and <filename
+ 		   class="headerfile">src/Utilities/ObserverEvent.h</filename>:
+      <type>Observable&lt;T&gt;</type>,
+      <type>SingleObserveable&lt;T&gt;</type>,
+      <type>Observer&lt;T&gt;</type>, and <type>ObserverEvent</type>
+      implement the observer pattern.  What is the observer pattern?
+      Where is this used in the code?</para>
+     </listitem>
+    </itemizedlist>
+ 
+   </chapter>
+ 
+  </part>
+ 
+ 
+  <appendix id="future_development">
+   <title>Future Development</title>
+ 
+   <section id="future_development-particles">
+    <title>Particles</title>
+ 
+    <para><filename
+ 		   class="libraryfile">docs/ParticlesDoc.txt</filename> has
+    out-of-date information.</para>
+ 
+    <para>See Section&nbsp;3.2.3 of
+    <filename>papers/pooma.ps</filename> for an out-of-date
+    description.</para>
+ 
+    <para><filename>papers/jvwr.ps</filename> concerns mainly
+    particles.  <filename>papers/8thSIAMPOOMAParticles.pdf</filename>,
+    by Julian Cummings and Bill Humphrey, concerns parallel particle
+    simulations.  <filename>papers/iscope98linac.pdf</filename>
+    describes a particle beam simulation using &pooma;; it mainly
+    concerns particles.</para>
+ 
+    <section>
+     <title>Particles</title>
+ 
+     <para>Do we want to include such a section?</para>
+ 
+     <para>Section&nbsp;3, "Sample Applications" of
+     <filename>papers/SiamOO98_paper.ps</filename> describes porting a
+     particle program written using High-Performance Fortran to
+     &pooma; and presumably why particles were added to &pooma;.  It
+     also describes <application>MC++</application>, a Monte Carlo
+     neutron transport code.</para>
+ 
+    </section>
+ 
+   </section>
+ 
+ 
+   <section id="future_development-composition_engine">
+    <title>Composition of &engine;s</title>
+ 
+    <para>The i,j-th element of the composition
+    <varname>a</varname>&compfn;<varname>b</varname> of two arrays
+    <varname>a</varname> and <varname>b</varname> equals a(b(i,j)).
+    The composition engine tagged <literal>IndirectionTag&lt;Array1,
+    Array2&gt;</literal>, defined in <filename
+ 					      class="headerfile">src/Engine/IndirectionEngine.h</filename> is
+    unfinished.</para>
+   </section>
+ 
+ 
+   <section id="future_development-container_consistency">
+    <title>Improving Consistency of Container Interfaces</title>
+ 
+    <section id="future_development-container_consistency-array_relations">
+     <title>Relations for &array;s</title>
+ 
+     <para>Do &array;s currently support relations?  If not, why not?
+    Should they be added?</para>
+    </section>
+ 
+    <section id="future_development-container_consistency-dimensions">
+     <title>Supporting the Same Number of Dimensions</title>
+ 
+     <para>&array; and &field; should support the same maximum number
+     of dimensions.  Currently, &array;s support seven dimensions and
+     &field;s support only three.  By definition, &dynamicarray;
+     supports only one dimension.</para>
+ 
+     <para>Relations for &array;s.</para>
+ 
+     <para>External guards for &array;s.</para>
+    </section>
+ 
+   </section>
+ 
+ 
+   <section id="future_development-where">
+    <title><function>where</function> Proxies</title>
+ 
+    <para>QUESTION: Do we even discuss this broken
+    <quote>feature</quote>?  Where is it used?  Some related code is in
+    <filename>src/Array/Array.h</filename>:2511&ndash;2520.</para>
+   </section>
+ 
+ 
+   <section id="future_development-long_term">
+    <title>Very Long Term Development Ideas</title>
+ 
+    <para>Describe how to write a new configuration file.</para>
+   </section>
+ 
+  </appendix>
+ 
+ 
+  <appendix id="installation">
+   <title>Obtaining and Installing &pooma;</title>
+ 
+   <para>ADD: Write this section, including extensive instructions
+   for Unix, MS Windows, and MacOS.  List the configuration options.
+   Be sure to describe configuring for parallel execution.</para>
+ 
+   <section id="installation-distributed_computing">
+    <title>Supporting Distributed Computation</title>
+ 
+    <para>To use multiple processors with &pooma; requires installing
+    the &cheetah; messaging library and an underlying messaging library
+    such as the Message Passing Interface (&mpi;) Communications
+    Library or the &mm; Shared Memory Library.  In this section, we
+    first describe how to install &mm;.  Read the section only if using
+    &mm;, not &mpi;.  Then we describe how to install &cheetah; and
+    configure &pooma; to use it.</para>
+ 
+    <section id="installation-distributed_computing-mm">
+     <title>Obtaining and Installing the &mm; Shared Memory Library</title>
+ 
+     <para>&cheetah;, and thus &pooma;, can use Ralf Engelschall's &mm;
+     Shared Memory Library to pass messages between processors.  For
+     example, the &author; uses this library on a two-processor
+     computer running &linux;.  The library, available at
+     http://www.engelschall.com/sw/mm/, is available for free and has
+     been successfully tested on a variety of Unix platforms.</para>
+ 
+     <para>We describe how to download and install the &mm; library.
+      <orderedlist spacing="compact">
+ 	<listitem>
+ 	 <para>Download the library from the &pooma; Download page
+          available off the &pooma; home page (&poomaHomePage;).</para>
+ 	</listitem>
+ 	<listitem>
+ 	 <para>Extract the source code using <command>tar xzvf
+          mm-1.1.3.tar.gz</command>.  Move into the resulting source
+          code directory <filename
+          class="directory">mm-1.1.3</filename>.</para>
+ 	</listitem>
+ 	<listitem>
+ 	 <para>Prepare to compile the source code by configuring it
+          using the <command>configure</command> command.  To change
+          the default installation directory <filename
+          class="directory">/usr/local</filename>, specify
+          <command>--prefix=<replaceable>directory</replaceable></command>
+          option.  The other configuration options can be listed by
+          specifying the <command>--help</command> option.  Since the
+          &author; prefers to keep all &pooma;-related code in his
+          <filename class="directory">pooma</filename>subdirectory, he
+          uses <command>./configure
+          --prefix=${HOME}/pooma/mm-1.1.3</command>.</para>
+ 	</listitem>
+ 	<listitem>
+ 	 <para>Create the library by issuing the <command>make</command>
+      command.  This compiles the source code using a &c; compiler.  To
+      use a different compiler than the &mm; configuration chooses, set
+      the <envar>CC</envar> to the compiler before configuring.</para>
+       </listitem>
+     <listitem>
+      <para>Optionally test the library by issuing the <command>make
+      test</command> command.  If successful, the penultimate line
+      should be <computeroutput>OK - ALL TESTS SUCCESSFULLY
+      PASSED</computeroutput>.</para>
+     </listitem>
+     <listitem>
+      <para>Install the &mm; Library by issuing the <command>make
+      install</command> command.  This copies the library files to the
+      installation directory.  The <filename
+      class="directory">mm-1.1.3</filename> directory containing the
+      source code may now be removed.</para>
+     </listitem>
+    </orderedlist>
+    </para>
+    </section>
+ 
+    
+    <section id="installation-distributed_computing-cheetah">
+     <title>Obtaining and Installing the &cheetah; Messaging Library</title>
+ 
+     <para>The &cheetah; Library decouples communication from
+     synchronization.  Using asynchronous messaging rather than
+     synchronous messaging permits a message sender to operate without
+     the cooperation of the message recipient.  Thus, implementing
+     message sending is simpler and processing is more efficiently
+     overlapped with it.  Remote method invocation is also supported.
+     The library was developed at the Los Alamos National Laboratory's
+     Advanced Computing Laboratory.</para>
+ 
+     <para>&cheetah;'s messaging is implemented using an underlying
+     messaging library such as the Message Passing Interface (&mpi;)
+     Communications Library (FIXME: xref linkend="mpi99", <ulink
+     url="http://www-unix.mcs.anl.gov/mpi/"></ulink>) or the &mm;
+     Shared Memory Library.  &mpi; works on a wide variety of platforms
+     and has achieved widespread usage.  &mm; works under Unix on any
+     computer with shared memory.  Both libraries are available for
+     free.  The instructions below work for whichever library you
+     choose.</para>
+ 
+     <para>We describe how to download and install &cheetah;.
+      <orderedlist spacing="compact">
+       <listitem>
+        <para>Download the library from the &pooma; Download page
+        available off the &pooma; home page (&poomaHomePage;).</para>
+       </listitem>
+       <listitem>
+        <para>Extract the source code using <command>tar xzvf
+        cheetah-1.0.tgz</command>.  Move into the resulting source code
+        directory <filename
+        class="directory">cheetah-1.0</filename>.</para>
+       </listitem>
+       <listitem>
+        <para>Edit a configuration file corresponding to your operating
+        system and compiler.  These <filename
+        class="libraryfile">.conf</filename> files are located in the
+        <filename class="directory">config</filename> directory.  For
+        example, to use &gcc; with the &linux; operating system, use
+        <filename
+        class="libraryfile">config/LINUXGCC.conf</filename>.</para>
+ 
+        <para>The configuration file usually does not need
+        modification.  However, if you are using &mm;, ensure
+        <varname>shmem_default_dir</varname> specifies its location.
+        For example, the &author; modified the value to
+        <statement>"/home/oldham/pooma/mm-1.1.3"</statement>.</para>
+       </listitem>
+       <listitem>
+        <para>Prepare to compile the source code by configuring it
+        using the <command>configure</command> command.  Specify the
+        configuration file using the <command>--arch</command> option.
+        Its argument should be the configuration file's name, omitting
+        its <filename class="libraryfile">.conf</filename> suffix.  For
+        example, <command>--arch LINUXGCC</command>.  Some other
+        options include
+        <variablelist>
+        <varlistentry>
+ 	<term>--help</term>
+ 	<listitem>
+ 	 <para>lists all the available options</para>
+ 	</listitem>
+        </varlistentry>
+        <varlistentry>
+ 	<term>--shmem --nompi</term>
+ 	<listitem>
+ 	 <para>indicates use of &mm;, not &mpi;</para>
+ 	</listitem>
+        </varlistentry>
+        <varlistentry>
+ 	<term>--mpi --noshmem</term>
+ 	<listitem>
+ 	 <para>indicates use of &mpi;, not &mm;</para>
+ 	</listitem>
+        </varlistentry>
+        <varlistentry>
+ 	<term>--opt</term>
+ 	<listitem>
+ 	 <para>causes the compiler to produce optimized source code</para>
+ 	</listitem>
+        </varlistentry>
+        <varlistentry>
+ 	<term>--noex</term>
+ 	<listitem>
+ 	 <para>prevents use of &cc; exceptions</para>
+ 	</listitem>
+        </varlistentry>
+        <varlistentry>
+ 	<term>--prefix <replaceable>directory</replaceable></term>
+ 	<listitem>
+ 	 <para>specifies the installation directory where the
+ 	   library will be copied rather than the default.</para>
+ 	</listitem>
+        </varlistentry>
+       </variablelist>
+         For example, the &author; uses <command>./configure --arch
+         LINUXGCC --shmem --nompi --noex --prefix
+         ${HOME}/pooma/cheetah-1.0 --opt</command>.  The
+         <command>--arch LINUXGCC</command> indicates use of &gcc;
+         under a &linux; operating system.  The &mm; library is used,
+         but &cc; exceptions are not.  The latter choice matches
+         &pooma;'s default choice.  The library will be installed in
+         the <filename
+ 		      class="directory">${HOME}/pooma/cheetah-1.0</filename>.
+         Finally, the library code will be optimized, hopefully running
+         faster than unoptimized code.</para>
+       </listitem>
+       <listitem>
+        <para>Follow the directions printed by
+        <command>configure</command>: Change directories to the
+        <filename class="directory">lib</filename> subdirectory named
+        by the <command>--arch</command> argument and then type
+        <command>make</command> to compile the source code and create
+        the library.</para>
+       </listitem>
+       <listitem>
+        <para>Optionally ensure the library works correctly by issuing
+        the <command>make tests</command> command.</para>
+       </listitem>
+       <listitem>
+        <para>Install the library by issuing the <command>make
+        install</command> command.  This copies the library files to
+        the installation directory.  The <filename
+        class="directory">cheetah-1.0</filename> directory containing
+        the source code may now be removed.</para>
+       </listitem>
+      </orderedlist>
+    </para>
+    </section>
+ 
+    <section id="installation-distributed_computing-pooma">
+     <title>Configuring &pooma; When Using &cheetah;</title>
+ 
+     <para>To use &pooma; with &cheetah;, one must tell &pooma; the
+     location of the &cheetah; library using the
+     <command>--messaging</command> configuration option.  To do this,
+      <orderedlist spacing="compact">
+       <listitem>
+        <para>Set the &cheetah; directory environment variable
+         <envar>CHEETAHDIR</envar> to the directory containing the
+         installed &cheetah; library.  For
+         example, <command>declare -x
+         CHEETAHDIR=${HOME}/pooma/cheetah-1.0</command> specifies the
+         installation directory used in the previous section.</para>
+       </listitem>
+       <listitem>
+        <para>When configuring &pooma;, specify the
+        <command>--messaging</command> option.  For example,
+        <command>./configure --arch LINUXgcc --opt
+        --messaging</command> configures for &linux;, &gcc;, and an
+        optimized library using &cheetah;.</para>
+       </listitem>
+      </orderedlist>
+     </para>
+    </section>
+ 
+ <!-- HERE -->
+   </section>
+  </appendix>
+ 
+ 
+  <appendix id="compilation_errors">
+   <title>Dealing with Compilation Errors</title>
+ 
+   <para>Base this low-priority section on <filename
+   class="libraryfile">errors.html</filename>.  QUESTION: Where is
+   <filename class="libraryfile">errors.html</filename>?</para>
+ 
+  </appendix>
+ 
+ 
+  <appendix id="tmp_appendix">
+   <title>TMP: Notes to Myself</title>
+ 
+   <section id="tmp-miscellaneous">
+    <title>Miscellaneous</title>
+ 
+    <orderedlist>
+     <listitem>
+      <para>QUESTION: How do I know when to use a type name versus just
+      the concept?  For example, when do I use <quote>array</quote>
+      versus &array;?</para>
+     </listitem>
+ 
+     <listitem>
+      <para>Krylov solvers are described in Section&nbsp;3.5.2 of
+      <filename>papers/pooma.ps</filename>.</para>
+     </listitem>
+ 
+     <listitem>
+      <para>Section&nbsp;5, "The Polygon Overlay Problem," describes
+      porting an <acronym>ANSI</acronym> &c; program to &pooma;.</para>
+     </listitem>
+ 
+     <listitem>
+      <para>A good example book: <emphasis>STL Tutorial and Reference
+      Guide: &cc; Programming with the Standard Template
+      Library</emphasis>, second edition, by David&nbsp;R. Musser,
+      Gillmer&nbsp;J. Derge, and Atul Sanai, ISBN 0-201-37923-6,
+      QA76.73.C153.M87 2001.</para>
+     </listitem>
+ 
+     <listitem>
+      <para>One STL reference book listed functions in margin notes,
+      easing finding material.  Do this.</para>
+     </listitem>
+ 
+     <listitem>
+      <para>QUESTION: Does Berna Massingill at Trinity University have
+      any interest ior access to any parallel computers?</para>
+     </listitem>
+    </orderedlist>
+ 
+   </section>
+ 
+ 
+   <section id="tmp-html_tutorials">
+    <title>Existing HTML Tutorials</title>
+ 
+    <para>All these tutorials are out-of-date, but the ideas and text
+    may still be relevant.</para>
+ 
+    <variablelist>
+     <varlistentry><term><filename
+ 				  class="libraryfile">index.html</filename></term>
+      <listitem><para>list of all tutorials.  No useful
+     material.</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+ 				  class="libraryfile">introduction.html</filename></term>
+      <listitem><para>data-parallel Laplace solver using Jacobi
+     iteration ala <command>Doof2d</command></para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+ 				  class="libraryfile">background.html</filename></term>
+      <listitem><para>short, indirect introduction to &pete;; parallel
+     execution model; &cc;; templates; &stl;; expression
+     templates</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+ 				  class="libraryfile">tut-01.html</filename></term>
+      <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+ 				  class="libraryfile">Layout.html</filename></term>
+      <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+ 				  class="libraryfile">parallelism.html</filename></term>
+      <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+ 				  class="libraryfile">self-test.html</filename></term>
+      <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">threading.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-03.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-04.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-05.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-06.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-07.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-08.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-09.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-10.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-11.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-12.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-13.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+    </variablelist>
+ 
+   </section>
+ 
+  </appendix>
+ 
+ 
+  <!-- Bibliography -->
+ 
+  <bibliography id="bibliography">
+   <title>Bibliography</title>
+ 
+   <para>FIXME: How do I process these entries?</para>
+ 
+   <biblioentry>
+    <abbrev>mpi99</abbrev>
+    <authorgroup>
+     <author>
+      <firstname>William</firstname><surname>Gropp</surname>
+     </author>
+     <author>
+      <firstname>Ewing</firstname><surname>Lusk</surname>
+     </author>
+     <author>
+      <firstname>Anthony</firstname><surname>Skjellum</surname>
+     </author>
+    </authorgroup>
+    <copyright>
+     <year>1999</year>
+     <holder>Massachusetts Institute of Technology</holder>
+    </copyright>
+    <isbn>0-262-57132-3</isbn>
+    <publisher>
+     <publishername>The MIT Press</publishername>
+     <address>Cambridge, MA</address>
+    </publisher>
+    <title>Using MPI</title>
+    <subtitle>Portable Parallel Programming with the Message-Passing Interface</subtitle>
+    <edition>second edition</edition>
+   </biblioentry>
+  </bibliography>
+ 
+ 
+  <!-- Glossary -->
+ 
+  <glossary id="glossary">
+   <title>Glossary</title>
+ 
+   <para>ADD: Make sure all entries are indexed and perhaps point back
+   to their first use.  WARNING: This is constructed by hand so it is
+   likely to be full of inconsistencies and errors.</para>
+ 
+   <glossdiv>
+    <title>S</title>
+ 
+    <glossentry id="glossary-suite_name">
+     <glossterm>Suite Name</glossterm>
+     <glossdef>
+      <para>An arbitrary string denoting a particular toolkit
+      configuration.  For example, the string
+      <quote>SUNKCC-debug</quote> might indicate a configuration for
+      the <trademark>Sun</trademark> <productname>Solaris</productname>
+      operating system and the &kcc; &cc; compiler with debugging
+      support.  By default, the suite name it is equal to the
+      configuration's architecture name.</para>
+     </glossdef>
+    </glossentry>
+   </glossdiv>
+  </glossary>
+  
+ 
+  <!-- Index -->
+ 
+   &genindex.sgm;
+ 
+ </book>
Index: docs/manual/figures/distributed.mp
===================================================================
RCS file: distributed.mp
diff -N distributed.mp
*** /dev/null	Fri Mar 23 21:37:44 2001
--- distributed.mp	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,195 ----
+ %% Oldham, Jeffrey D.
+ %% 2001Nov28
+ %% Pooma
+ 
+ %% Illustrations for Distributed Computing
+ 
+ %% Assumes TEX=latex.
+ 
+ input boxes;
+ 
+ verbatimtex
+ \documentclass[10pt]{article}
+ \begin{document}
+ etex
+ 
+ %% Parts of Distributed Computation
+ beginfig(101)
+   numeric unit; unit = 0.9cm;
+ 
+   %% Create the Container Storage Partition subfigure.
+   numeric arrayWidth; arrayWidth = 2; % as multiple of unit
+   numeric arrayHeight; arrayHeight = 4; % as multiple of unit
+   numeric guardWidth; guardWidth = 0.1; % as multiple of unit
+   numeric patchWidth; patchWidth = arrayWidth/3; % as multiple of unit
+   numeric patchHeight; patchHeight = arrayHeight/2; % as multiple of unit
+   numeric xPatchDistance; xPatchDistance = 0.9patchWidth; % as multiple of unit
+   numeric yPatchDistance; yPatchDistance = 0.5patchWidth; % as multiple of unit
+   numeric arrayPartitionDistance; arrayPartitionDistance = arrayWidth;
+ 				% distance between array and partition
+   numeric arrayLayoutDistance; arrayLayoutDistance = 0.5arrayHeight;
+ 				% distance between array and layout
+   numeric arrowLayoutDistance; arrowLayoutDistance = 0.5arrayLayoutDistance;
+ 				% distance between arrow and top of layout, not its label
+   numeric iota; iota = labeloffset;
+   numeric storageBoundaryWidth; storageBoundaryWidth = 1; % as multiple of unit
+ 				% gap between storage box and its internals
+   
+   % Create the Array.  Use box "a".
+   boxit.a();
+   a.ne - a.sw = unit*(arrayWidth,arrayHeight);
+ 
+   % Create the partition.  Use boxes "p[]".
+   for t = 0 upto 5:
+     boxit.p[t]();
+     p[t].ne - p[t].sw = unit*(1,1);
+   endfor;
+   for t = 0 upto 2:
+     p[t].sw=p[t+3].nw;
+     p[t].se=p[t+3].ne;
+     if t < 2:
+       p[t].ne = p[t+1].nw;
+       p[t+3].ne = p[t+4].nw;
+     fi
+   endfor;
+   boxit.pt(btex \begin{tabular}{c} + \\ external guard layers \\ + \\ internal guard layers \end{tabular} etex);
+   pt.n = p[4].s;
+ 
+   % Create the layout patches "l[]" and their guard layers "g[]".
+   for t = 0 upto 5:
+     boxit.l[t]();
+     boxit.g[t]();
+     l[t].ne - l[t].sw = unit*(patchWidth, patchHeight);
+     g[t].ne - l[t].ne = -(g[t].sw - l[t].sw) = unit*guardWidth*(1,1);
+   endfor
+   for t = 0 upto 2:
+     if t < 2:
+       g[t+1].nw - g[t].ne = unit*(xPatchDistance,0);
+     fi
+     g[t].sw - g[t+3].nw = unit*(0,yPatchDistance);
+   endfor;
+ 
+   % Create the storage equation boxes.
+   boxit.containerPlus(btex + etex);
+   boxit.containerArrow(btex $\Big\Downarrow$ etex);
+ 
+   % Position the storage pieces.
+   p[0].nw - a.ne = unit*(arrayPartitionDistance,0);
+   containerPlus.c = (xpart(0.5[a.ne,p[0].nw]), ypart(a.c));
+   containerArrow.c = (xpart(containerPlus.c), ypart(l[1].n) + unit*arrowLayoutDistance);
+   ypart(a.s - l[1].n) = unit*arrayLayoutDistance;
+   xpart(containerPlus.c - l[1].n) = 0;
+ 
+   % Create a boundary box around storage partition.
+   boxit.storageBoundary();
+   ypart(storageBoundary.n - a.n) =
+   ypart(l[4].s - storageBoundary.s) = unit*2storageBoundaryWidth;
+   xpart(a.w - storageBoundary.w) = unit*storageBoundaryWidth;
+   xpart(storageBoundary.e - pt.e) = unit*storageBoundaryWidth;
+   
+   %% Create the Computer Configuration subfigure. 
+   numeric configurationBoundaryWidth; configurationBoundaryWidth = storageBoundaryWidth;
+ 				% gap between computer configuration box and its internals
+ 
+   for t = 0 upto 2:
+     circleit.c[t]();
+     c[t].n - c[t].s = 1.3(0,ypart(g[0].ne - g[3].sw));
+     c[t].e - c[t].w = 1.5(xpart(g[0].ne - g[3].sw),0);
+   endfor
+   c[2].c - c[1].c = c[1].c - c[0].c = g[1].c - g[0].c;
+ 
+   boxit.configurationBoundary();
+   ypart(configurationBoundary.n - configurationBoundary.s) =
+   ypart(storageBoundary.n - storageBoundary.s);
+   xpart(configurationBoundary.e - configurationBoundary.w) =
+   xpart(c[2].e - c[0].w)+2*unit*configurationBoundaryWidth;
+   configurationBoundary.c = c[1].c;
+ 
+   %% Create the Computation Configuration subfigure.
+   % Create the patches.
+   for t = 0 upto 5:
+     boxit.L[t]();
+     boxit.G[t]();
+     L[t].ne - L[t].sw = unit*(patchWidth, patchHeight);
+     G[t].ne - L[t].ne = -(G[t].sw - L[t].sw) = unit*guardWidth*(1,1);
+   endfor
+   for t = 0 upto 2:
+     if t < 2:
+       G[t+1].nw - G[t].ne = unit*(xPatchDistance,0);
+     fi
+     G[t].sw - G[t+3].nw = unit*(0,yPatchDistance);
+   endfor;
+ 
+   % Create the contexts.
+   for t = 0 upto 2:
+     circleit.C[t]();
+     C[t].n - C[t].s = 1.3(0,ypart(G[0].ne - G[3].sw));
+     C[t].e - C[t].w = 1.5(xpart(G[0].ne - G[3].sw),0);
+   endfor
+   C[2].c - C[1].c = C[1].c - C[0].c = G[1].c - G[0].c;
+   C[0].c = 0.5[G[0].c,G[3].c];
+ 
+   %% Relate the subfigures.
+   numeric containerConfigurationDistance;
+   containerConfigurationDistance = arrayPartitionDistance;
+ 				% distance between container storage and computer configuration subfigures
+   numeric containerComputationDistance; containerComputationDistance = arrayLayoutDistance;
+ 				% distance between container storage subfigure and computation configuration subfigure
+   numeric arrowComputationDistance; arrowComputationDistance = arrowLayoutDistance;
+ 				% distance between arrow and top of computation configuration, not its label
+ 
+   boxit.figurePlus(btex + etex);
+   boxit.figureArrow(btex $\Big\Downarrow$ etex);
+ 
+   configurationBoundary.w - storageBoundary.e =
+   unit*(containerConfigurationDistance,0); %% HERE
+   figurePlus.c = 0.5[configurationBoundary.w, storageBoundary.e];
+   figureArrow.c = (xpart(0.5[configurationBoundary.e,storageBoundary.w]),
+     ypart(C[1].n) + unit*arrowComputationDistance);
+   
+   0.5[configurationBoundary.se,storageBoundary.sw] - C[1].n =
+   unit*(0,containerComputationDistance);
+   
+   %% Draw the Container Domain Partitioning structures.
+   drawboxed(a);  label.top(btex \begin{tabular}{c} container's\\ domain \end{tabular} etex, a.n);
+   for t = 0 upto 5:
+     drawboxed(p[t]);
+   endfor
+   drawunboxed(pt);
+   label.top(btex partition etex, p[1].n);
+   for t = 0 upto 5:
+     drawboxed(l[t],g[t]);
+   endfor
+   label.top(btex patches etex, g[1].n);
+   z0 = g[2].e + unit*(1,0);
+   drawarrow z0 -- (g[2].e+(iota,0));
+   label.rt(btex \begin{tabular}{l} patch with\\guard cells \end{tabular} etex, z0);
+   drawunboxed(containerPlus,containerArrow);
+   drawboxed(storageBoundary);
+   label.top(btex Partition Container's Domain etex, storageBoundary.n);
+ 
+   %% Draw the Computer Configuration structures.
+   for t = 0 upto 2:
+     drawboxed(c[t]);
+   endfor
+   label.top(btex contexts etex, c[1].n);
+   label.bot(btex \begin{tabular}{c} Each context has memory and\\ processors to execute a program. \end{tabular} etex, c[1].s);
+   drawboxed(configurationBoundary);
+   label.top(btex Computer Configuration etex, configurationBoundary.n);
+ 
+   %% Draw the Computer Computation structures.
+   for t = 0 upto 5:
+     drawboxed(L[t],G[t]);
+   endfor
+   for t = 0 upto 2:
+     drawboxed(C[t]);
+   endfor
+   label.top(btex Layout etex, C[1].n);
+   label.bot(btex Each context can contain several patches. etex, C[1].s);
+ 
+   %% Draw the subfigure relations structures.
+   drawunboxed(figurePlus,figureArrow);
+   label.rt(btex DistributedTag etex, figureArrow.e);
+ endfig;
+ 
+ bye
Index: docs/manual/figures/doof2d.mp
===================================================================
RCS file: doof2d.mp
diff -N doof2d.mp
*** /dev/null	Fri Mar 23 21:37:44 2001
--- doof2d.mp	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,257 ----
+ %% Oldham, Jeffrey D.
+ %% 2001Nov26
+ %% Pooma
+ 
+ %% Illustrations for the Tutorial Chapter (Chapter 2)
+ 
+ %% Assumes TEX=latex.
+ 
+ verbatimtex
+ \documentclass[10pt]{article}
+ \begin{document}
+ etex
+ 
+ % Draw a set of grid cells.
+ vardef drawGrid(expr nuCells, unit, llCorner) =
+   for i = 0 upto nuCells-1:
+     for j = 0 upto nuCells-1:
+       draw unitsquare scaled unit shifted (llCorner + unit*(i,j));
+     endfor
+   endfor
+ enddef;
+ 
+ % Label the specified grid, grid cell, or its edge.
+ % Place a value at the center of a grid cell.
+ vardef labelCell(expr lbl, xy, llCorner) =
+   label(lbl, llCorner + unit*(xy + 0.5*(1,1)));
+ enddef;
+ 
+ % Label the bottom of a grid cell.
+ vardef labelCellBottom(expr lbl, xy, llCorner) =
+   label.bot(lbl, llCorner + unit*(xy + 0.5*(1,0)));
+ enddef;
+ 
+ % Label the left side of a grid cell.
+ vardef labelCellLeft(expr lbl, xy, llCorner) =
+   label.lft(lbl, llCorner + unit*(xy + 0.5*(0,1)));
+ enddef;
+ 
+ % Label the top of a grid.
+ vardef labelGrid(expr lbl, nuCells, llCorner) =
+   label.top(lbl, llCorner + unit*(nuCells/2,nuCells));
+ enddef;
+ 
+ %% Global Declarations
+ numeric unit; unit = 0.9cm;	% width or height of an individual grid cell
+ 
+ 
+ %% Initial Configuration.
+ beginfig(201)
+   numeric nuCells; nuCells = 7;	% number of cells in each dimension
+ 				% This number should be odd.
+   % Draw the grid cells.
+   drawGrid(nuCells, unit, origin);
+ 
+   % Label the grid cells' values.
+   for i = 0 upto nuCells-1:
+     for j = 0 upto nuCells-1:
+       if ((i = nuCells div 2) and (j = nuCells div 2)):
+ 	labelCell(btex \footnotesize 1000.0 etex, (i,j), origin);
+       else:
+ 	labelCell(btex \footnotesize 0.0 etex, (i,j), origin);
+       fi
+     endfor
+   endfor
+ 
+   % Label the grid.
+   labelGrid(btex Array \texttt{b}: Initial Configuration etex, nuCells, origin);
+ endfig;
+ 
+ 
+ %% After the first averaging.
+ beginfig(202)
+   numeric nuCells; nuCells = 7;	% number of cells in each dimension
+ 				% This number should be odd.
+   % Draw the grid cells.
+   drawGrid(nuCells, unit, origin);
+ 
+   % Label the grid cells' values.
+   for i = 0, 1, nuCells-2, nuCells-1:
+     for j = 0 upto nuCells-1:
+       labelCell(btex \footnotesize 0.0 etex, (i,j), origin);
+     endfor
+   endfor
+   for j = 0, 1, nuCells-2, nuCells-1:
+     for i = 0 upto nuCells-1:
+       labelCell(btex \footnotesize 0.0 etex, (i,j), origin);
+     endfor
+   endfor
+   for i = (nuCells div 2)-1 upto (nuCells div 2)+1:
+     for j = (nuCells div 2)-1 upto (nuCells div 2)+1:
+       labelCell(btex \footnotesize 111.1 etex, (i, j), origin);
+     endfor
+   endfor
+   
+   % Label the grid.
+   labelGrid(btex Array \texttt{a}: After the first averaging etex, nuCells, origin);
+ endfig;
+ 
+ 
+ %% After the second averaging.
+ beginfig(203)
+   numeric nuCells; nuCells = 7;	% number of cells in each dimension
+ 				% This number should be odd.
+   % Draw the grid cells.
+   drawGrid(nuCells, unit, origin);
+ 
+   % Label the grid cells' values.
+   for i = 0, nuCells-1:
+     for j = 0 upto nuCells-1:
+       labelCell(btex \footnotesize 0.0 etex, (i,j), origin);
+     endfor
+   endfor
+   for j = 0, nuCells-1:
+     for i = 0 upto nuCells-1:
+       labelCell(btex \footnotesize 0.0 etex, (i,j), origin);
+     endfor
+   endfor
+   labelCell(btex \footnotesize 111.1 etex, (3,3), origin);
+   for t = (3,2), (4,3), (3,4), (2,3):
+     labelCell(btex \footnotesize 74.1 etex, t, origin);
+   endfor
+   for t = (2,2), (2,4), (4,4), (4,2):
+     labelCell(btex \footnotesize 49.4 etex, t, origin);
+   endfor
+   for t = (3,1), (5,3), (3,5), (1,3):
+     labelCell(btex \footnotesize 37.0 etex, t, origin);
+   endfor
+   for t = (1,2), (2,1), (4,1), (5,2), (5,4), (4,5), (2,5), (1,4):
+     labelCell(btex \footnotesize 24.7 etex, t, origin);
+   endfor
+   for t = (1,1), (5,1), (5,5), (1,5):
+     labelCell(btex \footnotesize 12.3 etex, t, origin);
+   endfor
+   
+   % Label the grid.
+   labelGrid(btex Array \texttt{b}: After the second averaging etex, nuCells, origin);
+ endfig;
+ 
+ 
+ %% Illustrate addition of arrays.
+ beginfig(210)
+   numeric nuCells; nuCells = 3;	% number of cells in each dimension
+ 				% This number should be odd.
+   numeric operatorWidth; operatorWidth = 1.5;
+   				% horizontal space for an operator as
+   				% a multiple of "unit"
+   
+   %% Determine the locations of the arrays.
+   z0 = origin;
+   z1 = z0 + unit * (nuCells+operatorWidth,0);
+   z2 - z1 = z1 - z0;
+ 
+   %% Draw the grid cells and the operators.
+   for t = 0 upto 2:
+     drawGrid(nuCells, unit, z[t]);
+   endfor
+   label(btex = etex, z1 + unit*(-0.9operatorWidth, 0.5nuCells));
+   label(btex + etex, z2 + unit*(-0.9operatorWidth, 0.5nuCells));
+   
+   %% Label the grid cells' values.
+   % Label b(I,J) grid values.
+   labelCell(btex \normalsize 9 etex, (0,0), z1);
+   labelCell(btex \normalsize 11 etex, (1,0), z1);
+   labelCell(btex \normalsize 13 etex, (2,0), z1);
+   labelCell(btex \normalsize 17 etex, (0,1), z1);
+   labelCell(btex \normalsize 19 etex, (1,1), z1);
+   labelCell(btex \normalsize 21 etex, (2,1), z1);
+   labelCell(btex \normalsize 25 etex, (0,2), z1);
+   labelCell(btex \normalsize 27 etex, (1,2), z1);
+   labelCell(btex \normalsize 29 etex, (2,2), z1);
+   % Label b(I+1,J-1) grid values.
+   labelCell(btex \normalsize 3 etex, (0,0), z2);
+   labelCell(btex \normalsize 5 etex, (1,0), z2);
+   labelCell(btex \normalsize 7 etex, (2,0), z2);
+   labelCell(btex \normalsize 11 etex, (0,1), z2);
+   labelCell(btex \normalsize 13 etex, (1,1), z2);
+   labelCell(btex \normalsize 15 etex, (2,1), z2);
+   labelCell(btex \normalsize 19 etex, (0,2), z2);
+   labelCell(btex \normalsize 21 etex, (1,2), z2);
+   labelCell(btex \normalsize 23 etex, (2,2), z2);
+   % Label b(I,J)+b(I+1,J-1) grid values.
+   labelCell(btex \normalsize 12 etex, (0,0), z0);
+   labelCell(btex \normalsize 16 etex, (1,0), z0);
+   labelCell(btex \normalsize 20 etex, (2,0), z0);
+   labelCell(btex \normalsize 28 etex, (0,1), z0);
+   labelCell(btex \normalsize 32 etex, (1,1), z0);
+   labelCell(btex \normalsize 36 etex, (2,1), z0);
+   labelCell(btex \normalsize 34 etex, (0,2), z0);
+   labelCell(btex \normalsize 38 etex, (1,2), z0);
+   labelCell(btex \normalsize 42 etex, (2,2), z0);
+ 
+   %% Label the indices.
+   % Label b(I,J) grid indices.
+   labelCellBottom(btex \footnotesize 1 etex, (0,0), z1);
+   labelCellBottom(btex \footnotesize 2 etex, (1,0), z1);
+   labelCellBottom(btex \footnotesize 3 etex, (2,0), z1);
+   labelCellLeft(btex \footnotesize 1 etex, (0,0), z1);
+   labelCellLeft(btex \footnotesize 2 etex, (0,1), z1);
+   labelCellLeft(btex \footnotesize 3 etex, (0,2), z1);
+   % Label b(I+1,J-1) grid indices.
+   labelCellBottom(btex \footnotesize 2 etex, (0,0), z2);
+   labelCellBottom(btex \footnotesize 3 etex, (1,0), z2);
+   labelCellBottom(btex \footnotesize 4 etex, (2,0), z2);
+   labelCellLeft(btex \footnotesize 0 etex, (0,0), z2);
+   labelCellLeft(btex \footnotesize 1 etex, (0,1), z2);
+   labelCellLeft(btex \footnotesize 2 etex, (0,2), z2);
+   
+   %% Label the grids.
+   labelGrid(btex $b(I,J)+b(I+1,J-1)$ etex, nuCells, z0);
+   labelGrid(btex $b(I,J)$ etex, nuCells, z1);
+   labelGrid(btex $b(I+1,J-1)$ etex, nuCells, z2);
+ endfig;
+ 
+ 
+ %% Illustrate application of a stencil.
+ beginfig(211)
+   numeric nuCells; nuCells = 5;	% number of cells in each dimension
+   numeric nuStencilCells; nuStencilCells = 3;
+ 				% number of stencil cells in each dimension
+   numeric stencilMultiple; stencilMultiple = 0.1;
+ 				% small multiple to make it visible
+   
+   % Draw the grid cells.
+   drawGrid(nuCells, unit, origin);
+ 
+   % Draw the stencil.
+   draw unitsquare scaled ((nuStencilCells-2stencilMultiple) * unit) shifted (unit*(stencilMultiple*(1,1)+(0,2)));
+   draw (unitsquare scaled ((1-stencilMultiple) * unit) shifted (unit*(0.5*stencilMultiple*(1,1)+(1,3)))) dashed evenly;
+ 
+   % Label the extents.
+   picture lbl;
+   ahlength := 0.4unit;
+   drawarrow unit*(2,4) -- unit*(3,5);
+   lbl = thelabel.lrt(btex \scriptsize upperExtent etex, unit*0.5[(2,4),(3,5)]);
+   unfill bbox lbl; draw lbl;
+   drawarrow unit*(1,3) -- unit*(0,2);
+   lbl := thelabel.lrt(btex \scriptsize lowerExtent etex, unit*0.5[(1,3),(0,2)]);
+   unfill bbox lbl; draw lbl;
+   
+   % Label the indices.
+   labelCellBottom(btex \footnotesize 0 etex, (0,0), origin);
+   labelCellBottom(btex \footnotesize 1 etex, (1,0), origin);
+   labelCellBottom(btex \footnotesize 2 etex, (2,0), origin);
+   labelCellBottom(btex \footnotesize 3 etex, (3,0), origin);
+   labelCellBottom(btex \footnotesize 4 etex, (4,0), origin);
+   labelCellLeft(btex \footnotesize 0 etex, (0,0), origin);
+   labelCellLeft(btex \footnotesize 1 etex, (0,1), origin);
+   labelCellLeft(btex \footnotesize 2 etex, (0,2), origin);
+   labelCellLeft(btex \footnotesize 3 etex, (0,3), origin);
+   labelCellLeft(btex \footnotesize 4 etex, (0,4), origin);
+ 
+   % Label the grid.
+   labelGrid(btex Applying a Stencil to Position (1,3) etex, nuCells, origin);
+   
+ endfig;
+ 
+ bye
Index: docs/manual/figures/makefile
===================================================================
RCS file: makefile
diff -N makefile
*** /dev/null	Fri Mar 23 21:37:44 2001
--- makefile	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,69 ----
+ ### Oldham, Jeffrey D.
+ ### 1997 Dec 26
+ ### misc
+ ###
+ ### LaTeX -> PostScript/PDF/WWW
+ ### XML -> TeX/DVI/PS/PDF
+ 
+ # Definitions for PostScript and WWW Creation
+ TEX=		latex
+ WWWHOMEDIR=	/u/oldham/www
+ LATEX2HTML=	latex2html
+ BASICLATEX2HTMLOPTIONS=	-info "" -no_footnode -no_math -html_version 3.2,math
+ #LATEX2HTMLOPTIONS=	-local_icons -split +1 $(BASICLATEX2HTMLOPTIONS)
+ LATEX2HTMLOPTIONS=	-no_navigation -split 0 $(BASICLATEX2HTMLOPTIONS)
+ MPOST=		mpost
+ 
+ # Definitions for Jade.
+ JADEDIR=		/usr/lib/sgml/stylesheets/docbook
+ PRINTDOCBOOKDSL=	print/docbook.dsl
+ HTMLDOCBOOKDSL=		html/docbook.dsl
+ XML=			dtds/decls/xml.dcl
+ INDEXOPTIONS=		-t 'Index' -i 'index' -g -p
+ 
+ CXXFLAGS= -g -Wall -pedantic -W -Wstrict-prototypes -Wpointer-arith -Wbad-function-cast -Wcast-align -Wconversion -Wnested-externs -Wundef -Winline -static
+ 
+ all: outline.ps
+ 
+ %.all:	%.ps %.pdf %.html
+ 	chmod 644 $*.ps $*.pdf
+ 	mv $*.ps $*.pdf $*
+ 
+ %.dvi:	%.ltx
+ 	$(TEX) $<
+ 	# bibtex $*
+ 	# $(TEX) $<
+ 	$(TEX) $<
+ 
+ %.ps:	%.dvi
+ 	dvips -t letter $< -o
+ 
+ %.pdf.ltx:	%.ltx
+ 	sed -e 's/^%\\usepackage{times}/\\usepackage{times}/' $< > $@
+ 
+ %.pdf:	%.pdf.ps
+ 	ps2pdf $< $@
+ 
+ # This rule assumes index creation.
+ %.dvi:	%.xml genindex.sgm
+ 	jade -D$(JADEDIR) -t sgml -d $(HTMLDOCBOOKDSL) -V html-index $(XML) $<
+ 	perl collateindex.pl $(INDEXOPTIONS) -o genindex.sgm HTML.index
+ 	jade -D$(JADEDIR) -t tex -d $(PRINTDOCBOOKDSL) $(XML) $< && jadetex $*.tex && jadetex $*.tex && jadetex $*.tex
+ 
+ genindex.sgm:
+ 	perl collateindex.pl $(INDEXOPTIONS) -N -o $@
+ 
+ %.html:	%.xml
+ 	jade -D$(JADEDIR) -t sgml -d $(HTMLDOCBOOKDSL) $(XML) $<
+ 
+ %.pdf:	%.xml
+ 	jade -D$(JADEDIR) -t tex -d $(PRINTDOCBOOKDSL) $(XML) $< && pdfjadetex $*.tex && pdfjadetex $*.tex
+ 
+ mproof-%.ps:	%.mp
+ 	declare -x TEX=latex && $(MPOST) $< && tex mproof.tex $*.[0-9]* && dvips mproof.dvi -o $@
+ 
+ %.txt:	%.ltx
+ 	detex $< > $@
+ 
+ clean:
+ 	rm -f *.dvi *.aux *.log *.toc *.bak *.blg *.bbl *.glo *.idx *.lof *.lot *.htm *.mpx mpxerr.tex HTML.index outline.tex
Index: docs/manual/programs/Doof2d-Array-distributed-annotated.patch
===================================================================
RCS file: Doof2d-Array-distributed-annotated.patch
diff -N Doof2d-Array-distributed-annotated.patch
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-Array-distributed-annotated.patch	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,162 ----
+ *** Doof2d-Array-distributed.cpp	Wed Nov 28 07:46:56 2001
+ --- Doof2d-Array-distributed-annotated.cpp	Wed Nov 28 07:53:31 2001
+ ***************
+ *** 1,4 ****
+ ! #include <iostream>		// has std::cout, ...
+ ! #include <stdlib.h>		// has EXIT_SUCCESS
+   #include "Pooma/Arrays.h"	// has Pooma's Array
+   
+ --- 1,5 ----
+ ! <programlisting id="tutorial-array_distributed-doof2d-program" linenumbering="numbered" format="linespecific">
+ ! #include &lt;iostream&gt;		// has std::cout, ...
+ ! #include &lt;stdlib.h&gt;		// has EXIT_SUCCESS
+   #include "Pooma/Arrays.h"	// has Pooma's Array
+   
+ ***************
+ *** 15,19 ****
+     // (i,j).  The "C" template parameter permits use of this stencil
+     // operator with both Arrays and Fields.
+ !   template <class C>
+     inline
+     typename C::Element_t
+ --- 16,20 ----
+     // (i,j).  The "C" template parameter permits use of this stencil
+     // operator with both Arrays and Fields.
+ !   template &lt;class C&gt;
+     inline
+     typename C::Element_t
+ ***************
+ *** 40,52 ****
+     Pooma::initialize(argc,argv);
+     
+ !   // Ask the user for the number of processors.
+     long nuProcessors;
+ !   std::cout << "Please enter the number of processors: ";
+ !   std::cin >> nuProcessors;
+   
+     // Ask the user for the number of averagings.
+     long nuAveragings, nuIterations;
+ !   std::cout << "Please enter the number of averagings: ";
+ !   std::cin >> nuAveragings;
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ --- 41,53 ----
+     Pooma::initialize(argc,argv);
+     
+ !   // Ask the user for the number of processors.  <co id="tutorial-array_distributed-doof2d-nuprocessors"></co>
+     long nuProcessors;
+ !   std::cout &lt;&lt; "Please enter the number of processors: ";
+ !   std::cin &gt;&gt; nuProcessors;
+   
+     // Ask the user for the number of averagings.
+     long nuAveragings, nuIterations;
+ !   std::cout &lt;&lt; "Please enter the number of averagings: ";
+ !   std::cin &gt;&gt; nuAveragings;
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ ***************
+ *** 54,67 ****
+     // the grid.
+     long n;
+ !   std::cout << "Please enter the array size: ";
+ !   std::cin >> n;
+   
+     // Specify the arrays' domains [0,n) x [0,n).
+ !   Interval<1> N(0, n-1);
+ !   Interval<2> vertDomain(N, N);
+   
+     // Set up interior domains [1,n-1) x [1,n-1) for computation.
+ !   Interval<1> I(1,n-2);
+ !   Interval<2> interiorDomain(I,I);
+   
+     // Create the distributed arrays.
+ --- 55,68 ----
+     // the grid.
+     long n;
+ !   std::cout &lt;&lt; "Please enter the array size: ";
+ !   std::cin &gt;&gt; n;
+   
+     // Specify the arrays' domains [0,n) x [0,n).
+ !   Interval&lt;1&gt; N(0, n-1);
+ !   Interval&lt;2&gt; vertDomain(N, N);
+   
+     // Set up interior domains [1,n-1) x [1,n-1) for computation.
+ !   Interval&lt;1&gt; I(1,n-2);
+ !   Interval&lt;2&gt; interiorDomain(I,I);
+   
+     // Create the distributed arrays.
+ ***************
+ *** 70,85 ****
+     // dimension.  Guard layers optimize communication between patches.
+     // Internal guards surround each patch.  External guards surround
+ !   // the entire array domain.
+ !   UniformGridPartition<2> partition(Loc<2>(nuProcessors, nuProcessors),
+ ! 				    GuardLayers<2>(1),  // internal
+ ! 				    GuardLayers<2>(0)); // external
+ !   UniformGridLayout<2> layout(vertDomain, partition, DistributedTag());
+   
+     // The template parameters indicate 2 dimensions and a 'double'
+     // element type.  MultiPatch indicates multiple computation patches,
+     // i.e., distributed computation.  The UniformTag indicates the
+ !   // patches should have the same size.  Each patch has Brick type.
+ !   Array<2, double, MultiPatch<UniformTag, Remote<Brick> > > a(layout);
+ !   Array<2, double, MultiPatch<UniformTag, Remote<Brick> > > b(layout);
+   
+     // Set up the initial conditions.
+ --- 71,86 ----
+     // dimension.  Guard layers optimize communication between patches.
+     // Internal guards surround each patch.  External guards surround
+ !   // the entire array domain.  <co id="tutorial-array_distributed-doof2d-layout"></co>
+ !   UniformGridPartition&lt;2&gt; partition(Loc&lt;2&gt;(nuProcessors, nuProcessors),
+ ! 				    GuardLayers&lt;2&gt;(1),  // internal
+ ! 				    GuardLayers&lt;2&gt;(0)); // external
+ !   UniformGridLayout&lt;2&gt; layout(vertDomain, partition, DistributedTag());
+   
+     // The template parameters indicate 2 dimensions and a 'double'
+     // element type.  MultiPatch indicates multiple computation patches,
+     // i.e., distributed computation.  The UniformTag indicates the
+ !   // patches should have the same size.  Each patch has Brick type.  <co id="tutorial-array_distributed-doof2d-remote"></co>
+ !   Array&lt;2, double, MultiPatch&lt;UniformTag, Remote&lt;Brick&gt; &gt; &gt; a(layout);
+ !   Array&lt;2, double, MultiPatch&lt;UniformTag, Remote&lt;Brick&gt; &gt; &gt; b(layout);
+   
+     // Set up the initial conditions.
+ ***************
+ *** 89,97 ****
+   
+     // Create the stencil performing the computation.
+ !   Stencil<DoofNinePt> stencil;
+   
+     // Perform the simulation.
+ !   for (int k = 0; k < nuIterations; ++k) {
+ !     // Read from b.  Write to a.
+       a(interiorDomain) = stencil(b, interiorDomain);
+   
+ --- 90,98 ----
+   
+     // Create the stencil performing the computation.
+ !   Stencil&lt;DoofNinePt&gt; stencil;
+   
+     // Perform the simulation.
+ !   for (int k = 0; k &lt; nuIterations; ++k) {
+ !     // Read from b.  Write to a.  <co id="tutorial-array_distributed-doof2d-first_write"></co>
+       a(interiorDomain) = stencil(b, interiorDomain);
+   
+ ***************
+ *** 102,106 ****
+     // Print out the final central value.
+     Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+ !   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
+   
+     // The arrays are automatically deallocated.
+ --- 103,107 ----
+     // Print out the final central value.
+     Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+ !   std::cout &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
+   
+     // The arrays are automatically deallocated.
+ ***************
+ *** 110,111 ****
+ --- 111,113 ----
+     return EXIT_SUCCESS;
+   }
+ + </programlisting>
Index: docs/manual/programs/Doof2d-Array-element-annotated.patch
===================================================================
RCS file: Doof2d-Array-element-annotated.patch
diff -N Doof2d-Array-element-annotated.patch
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-Array-element-annotated.patch	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,144 ----
+ *** Doof2d-Array-element.cpp	Tue Nov 27 11:04:04 2001
+ --- Doof2d-Array-element-annotated.cpp	Tue Nov 27 12:06:32 2001
+ ***************
+ *** 1,5 ****
+ ! #include <iostream>		// has std::cout, ...
+ ! #include <stdlib.h>		// has EXIT_SUCCESS
+ ! #include "Pooma/Arrays.h"	// has Pooma's Array
+   
+   // Doof2d: Pooma Arrays, element-wise implementation
+ --- 1,6 ----
+ ! <programlisting id="tutorial-array_elementwise-doof2d-program" linenumbering="numbered" format="linespecific">
+ ! #include &lt;iostream&gt;		// has std::cout, ...
+ ! #include &lt;stdlib.h&gt;		// has EXIT_SUCCESS
+ ! #include "Pooma/Arrays.h"	// has Pooma's Array  <co id="tutorial-array_elementwise-doof2d-header"></co>
+   
+   // Doof2d: Pooma Arrays, element-wise implementation
+ ***************
+ *** 7,17 ****
+   int main(int argc, char *argv[])
+   {
+ !   // Prepare the Pooma library for execution.
+     Pooma::initialize(argc,argv);
+     
+     // Ask the user for the number of averagings.
+     long nuAveragings, nuIterations;
+ !   std::cout << "Please enter the number of averagings: ";
+ !   std::cin >> nuAveragings;
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ --- 8,18 ----
+   int main(int argc, char *argv[])
+   {
+ !   // Prepare the Pooma library for execution.  <co id="tutorial-array_elementwise-doof2d-pooma_initialize"></co>
+     Pooma::initialize(argc,argv);
+     
+     // Ask the user for the number of averagings.
+     long nuAveragings, nuIterations;
+ !   std::cout &lt;&lt; "Please enter the number of averagings: ";
+ !   std::cin &gt;&gt; nuAveragings;
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ ***************
+ *** 19,37 ****
+     // the grid.
+     long n;
+ !   std::cout << "Please enter the array size: ";
+ !   std::cin >> n;
+   
+ !   // Specify the arrays' domains [0,n) x [0,n).
+ !   Interval<1> N(0, n-1);
+ !   Interval<2> vertDomain(N, N);
+   
+ !   // Create the arrays.
+     // The template parameters indicate 2 dimensions, a 'double' element
+     // type, and ordinary 'Brick' storage.
+ !   Array<2, double, Brick> a(vertDomain);
+ !   Array<2, double, Brick> b(vertDomain);
+   
+     // Set up the initial conditions.
+ !   // All grid values should be zero except for the central value.
+     a = b = 0.0;
+     b(n/2,n/2) = 1000.0;
+ --- 20,38 ----
+     // the grid.
+     long n;
+ !   std::cout &lt;&lt; "Please enter the array size: ";
+ !   std::cin &gt;&gt; n;
+   
+ !   // Specify the arrays' domains [0,n) x [0,n).  <co id="tutorial-array_elementwise-doof2d-domain"></co>
+ !   Interval&lt;1&gt; N(0, n-1);
+ !   Interval&lt;2&gt; vertDomain(N, N);
+   
+ !   // Create the arrays.  <co id="tutorial-array_elementwise-doof2d-array_creation"></co>
+     // The template parameters indicate 2 dimensions, a 'double' element
+     // type, and ordinary 'Brick' storage.
+ !   Array&lt;2, double, Brick&gt; a(vertDomain);
+ !   Array&lt;2, double, Brick&gt; b(vertDomain);
+   
+     // Set up the initial conditions.
+ !   // All grid values should be zero except for the central value.  <co id="tutorial-array_elementwise-doof2d-initialization"></co>
+     a = b = 0.0;
+     b(n/2,n/2) = 1000.0;
+ ***************
+ *** 41,49 ****
+   
+     // Perform the simulation.
+ !   for (int k = 0; k < nuIterations; ++k) {
+       // Read from b.  Write to a.
+ !     for (int j = 1; j < n-1; j++)
+ !       for (int i = 1; i < n-1; i++)
+ !         a(i,j) = weight *
+             (b(i+1,j+1) + b(i+1,j  ) + b(i+1,j-1) +
+              b(i  ,j+1) + b(i  ,j  ) + b(i  ,j-1) +
+ --- 42,50 ----
+   
+     // Perform the simulation.
+ !   for (int k = 0; k &lt; nuIterations; ++k) {
+       // Read from b.  Write to a.
+ !     for (int j = 1; j &lt; n-1; j++)
+ !       for (int i = 1; i &lt; n-1; i++)
+ !         a(i,j) = weight *  <co id="tutorial-array_elementwise-doof2d-first_write"></co>
+             (b(i+1,j+1) + b(i+1,j  ) + b(i+1,j-1) +
+              b(i  ,j+1) + b(i  ,j  ) + b(i  ,j-1) +
+ ***************
+ *** 51,56 ****
+   
+       // Read from a.  Write to b.
+ !     for (int j = 1; j < n-1; j++)
+ !       for (int i = 1; i < n-1; i++)
+           b(i,j) = weight *
+             (a(i+1,j+1) + a(i+1,j  ) + a(i+1,j-1) +
+ --- 52,57 ----
+   
+       // Read from a.  Write to b.
+ !     for (int j = 1; j &lt; n-1; j++)
+ !       for (int i = 1; i &lt; n-1; i++)
+           b(i,j) = weight *
+             (a(i+1,j+1) + a(i+1,j  ) + a(i+1,j-1) +
+ ***************
+ *** 60,70 ****
+   
+     // Print out the final central value.
+ !   Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+ !   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
+   
+ !   // The arrays are automatically deallocated.
+   
+ !   // Tell the Pooma library execution has finished.
+     Pooma::finalize();
+     return EXIT_SUCCESS;
+   }
+ --- 61,72 ----
+   
+     // Print out the final central value.
+ !   Pooma::blockAndEvaluate();	// Ensure all computation has finished.  <co id="tutorial-array_elementwise-doof2d-blockAndEvaluate"></co>
+ !   std::cout &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
+   
+ !   // The arrays are automatically deallocated.  <co id="tutorial-array_elementwise-doof2d-deallocation"></co>
+   
+ !   // Tell the Pooma library execution has finished.  <co id="tutorial-array_elementwise-doof2d-pooma_finish"></co>
+     Pooma::finalize();
+     return EXIT_SUCCESS;
+   }
+ + </programlisting>
Index: docs/manual/programs/Doof2d-Array-parallel-annotated.patch
===================================================================
RCS file: Doof2d-Array-parallel-annotated.patch
diff -N Doof2d-Array-parallel-annotated.patch
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-Array-parallel-annotated.patch	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,106 ----
+ *** Doof2d-Array-parallel.cpp	Tue Nov 27 13:00:09 2001
+ --- Doof2d-Array-parallel-annotated.cpp	Tue Nov 27 14:07:07 2001
+ ***************
+ *** 1,4 ****
+ ! #include <iostream>		// has std::cout, ...
+ ! #include <stdlib.h>		// has EXIT_SUCCESS
+   #include "Pooma/Arrays.h"	// has Pooma's Array
+   
+ --- 1,5 ----
+ ! <programlisting id="tutorial-array_parallel-doof2d-program" linenumbering="numbered" format="linespecific">
+ ! #include &lt;iostream&gt;		// has std::cout, ...
+ ! #include &lt;stdlib.h&gt;		// has EXIT_SUCCESS
+   #include "Pooma/Arrays.h"	// has Pooma's Array
+   
+ ***************
+ *** 12,17 ****
+     // Ask the user for the number of averagings.
+     long nuAveragings, nuIterations;
+ !   std::cout << "Please enter the number of averagings: ";
+ !   std::cin >> nuAveragings;
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ --- 13,18 ----
+     // Ask the user for the number of averagings.
+     long nuAveragings, nuIterations;
+ !   std::cout &lt;&lt; "Please enter the number of averagings: ";
+ !   std::cin &gt;&gt; nuAveragings;
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ ***************
+ *** 19,38 ****
+     // the grid.
+     long n;
+ !   std::cout << "Please enter the array size: ";
+ !   std::cin >> n;
+   
+     // Specify the arrays' domains [0,n) x [0,n).
+ !   Interval<1> N(0, n-1);
+ !   Interval<2> vertDomain(N, N);
+   
+ !   // Set up interior domains [1,n-1) x [1,n-1) for computation.
+ !   Interval<1> I(1,n-2);
+ !   Interval<1> J(1,n-2);
+   
+     // Create the arrays.
+     // The template parameters indicate 2 dimensions, a 'double' element
+     // type, and ordinary 'Brick' storage.
+ !   Array<2, double, Brick> a(vertDomain);
+ !   Array<2, double, Brick> b(vertDomain);
+   
+     // Set up the initial conditions.
+ --- 20,39 ----
+     // the grid.
+     long n;
+ !   std::cout &lt;&lt; "Please enter the array size: ";
+ !   std::cin &gt;&gt; n;
+   
+     // Specify the arrays' domains [0,n) x [0,n).
+ !   Interval&lt;1&gt; N(0, n-1);
+ !   Interval&lt;2&gt; vertDomain(N, N);
+   
+ !   // Set up interior domains [1,n-1) x [1,n-1) for computation.  <co id="tutorial-array_parallel-doof2d-innerdomain"></co>
+ !   Interval&lt;1&gt; I(1,n-2);
+ !   Interval&lt;1&gt; J(1,n-2);
+   
+     // Create the arrays.
+     // The template parameters indicate 2 dimensions, a 'double' element
+     // type, and ordinary 'Brick' storage.
+ !   Array&lt;2, double, Brick&gt; a(vertDomain);
+ !   Array&lt;2, double, Brick&gt; b(vertDomain);
+   
+     // Set up the initial conditions.
+ ***************
+ *** 45,50 ****
+   
+     // Perform the simulation.
+ !   for (int k = 0; k < nuIterations; ++k) {
+ !     // Read from b.  Write to a.
+       a(I,J) = weight *
+         (b(I+1,J+1) + b(I+1,J  ) + b(I+1,J-1) +
+ --- 46,51 ----
+   
+     // Perform the simulation.
+ !   for (int k = 0; k &lt; nuIterations; ++k) {
+ !     // Read from b.  Write to a.  <co id="tutorial-array_parallel-doof2d-first_write"></co>
+       a(I,J) = weight *
+         (b(I+1,J+1) + b(I+1,J  ) + b(I+1,J-1) +
+ ***************
+ *** 61,65 ****
+     // Print out the final central value.
+     Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+ !   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
+   
+     // The arrays are automatically deallocated.
+ --- 62,66 ----
+     // Print out the final central value.
+     Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+ !   std::cout &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
+   
+     // The arrays are automatically deallocated.
+ ***************
+ *** 69,70 ****
+ --- 70,72 ----
+     return EXIT_SUCCESS;
+   }
+ + </programlisting>
Index: docs/manual/programs/Doof2d-Array-stencil-annotated.patch
===================================================================
RCS file: Doof2d-Array-stencil-annotated.patch
diff -N Doof2d-Array-stencil-annotated.patch
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-Array-stencil-annotated.patch	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,152 ----
+ *** Doof2d-Array-stencil.cpp	Tue Nov 27 17:23:41 2001
+ --- Doof2d-Array-stencil-annotated.cpp	Tue Nov 27 17:36:13 2001
+ ***************
+ *** 1,9 ****
+ ! #include <iostream>		// has std::cout, ...
+ ! #include <stdlib.h>		// has EXIT_SUCCESS
+   #include "Pooma/Arrays.h"	// has Pooma's Array
+   
+   // Doof2d: Pooma Arrays, stencil implementation
+   
+ ! // Define the stencil class performing the computation.
+   class DoofNinePt
+   {
+ --- 1,10 ----
+ ! <programlisting id="tutorial-array_stencil-doof2d-program" linenumbering="numbered" format="linespecific">
+ ! #include &lt;iostream&gt;		// has std::cout, ...
+ ! #include &lt;stdlib.h&gt;		// has EXIT_SUCCESS
+   #include "Pooma/Arrays.h"	// has Pooma's Array
+   
+   // Doof2d: Pooma Arrays, stencil implementation
+   
+ ! // Define the stencil class performing the computation.  <co id="tutorial-array_stencil-doof2d-stencil"></co>
+   class DoofNinePt
+   {
+ ***************
+ *** 14,19 ****
+     // This stencil operator is applied to each interior domain position
+     // (i,j).  The "C" template parameter permits use of this stencil
+ !   // operator with both Arrays and Fields.
+ !   template <class C>
+     inline
+     typename C::Element_t
+ --- 15,20 ----
+     // This stencil operator is applied to each interior domain position
+     // (i,j).  The "C" template parameter permits use of this stencil
+ !   // operator with both Arrays and Fields.  <co id="tutorial-array_stencil-doof2d-stencil_operator"></co>
+ !   template &lt;class C&gt;
+     inline
+     typename C::Element_t
+ ***************
+ *** 26,30 ****
+     }
+   
+ !   inline int lowerExtent(int) const { return 1; }
+     inline int upperExtent(int) const { return 1; }
+   
+ --- 27,31 ----
+     }
+   
+ !   inline int lowerExtent(int) const { return 1; }  <co id="tutorial-array_stencil-doof2d-stencil_extent"></co>
+     inline int upperExtent(int) const { return 1; }
+   
+ ***************
+ *** 42,47 ****
+     // Ask the user for the number of averagings.
+     long nuAveragings, nuIterations;
+ !   std::cout << "Please enter the number of averagings: ";
+ !   std::cin >> nuAveragings;
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ --- 43,48 ----
+     // Ask the user for the number of averagings.
+     long nuAveragings, nuIterations;
+ !   std::cout &lt;&lt; "Please enter the number of averagings: ";
+ !   std::cin &gt;&gt; nuAveragings;
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ ***************
+ *** 49,68 ****
+     // the grid.
+     long n;
+ !   std::cout << "Please enter the array size: ";
+ !   std::cin >> n;
+   
+     // Specify the arrays' domains [0,n) x [0,n).
+ !   Interval<1> N(0, n-1);
+ !   Interval<2> vertDomain(N, N);
+   
+     // Set up interior domains [1,n-1) x [1,n-1) for computation.
+ !   Interval<1> I(1,n-2);
+ !   Interval<2> interiorDomain(I,I);
+   
+     // Create the arrays.
+     // The template parameters indicate 2 dimensions, a 'double' element
+     // type, and ordinary 'Brick' storage.
+ !   Array<2, double, Brick> a(vertDomain);
+ !   Array<2, double, Brick> b(vertDomain);
+   
+     // Set up the initial conditions.
+ --- 50,69 ----
+     // the grid.
+     long n;
+ !   std::cout &lt;&lt; "Please enter the array size: ";
+ !   std::cin &gt;&gt; n;
+   
+     // Specify the arrays' domains [0,n) x [0,n).
+ !   Interval&lt;1&gt; N(0, n-1);
+ !   Interval&lt;2&gt; vertDomain(N, N);
+   
+     // Set up interior domains [1,n-1) x [1,n-1) for computation.
+ !   Interval&lt;1&gt; I(1,n-2);
+ !   Interval&lt;2&gt; interiorDomain(I,I);
+   
+     // Create the arrays.
+     // The template parameters indicate 2 dimensions, a 'double' element
+     // type, and ordinary 'Brick' storage.
+ !   Array&lt;2, double, Brick&gt; a(vertDomain);
+ !   Array&lt;2, double, Brick&gt; b(vertDomain);
+   
+     // Set up the initial conditions.
+ ***************
+ *** 71,80 ****
+     b(n/2,n/2) = 1000.0;
+   
+ !   // Create the stencil performing the computation.
+ !   Stencil<DoofNinePt> stencil;
+   
+     // Perform the simulation.
+ !   for (int k = 0; k < nuIterations; ++k) {
+ !     // Read from b.  Write to a.
+       a(interiorDomain) = stencil(b, interiorDomain);
+   
+ --- 72,81 ----
+     b(n/2,n/2) = 1000.0;
+   
+ !   // Create the stencil performing the computation.  <co id="tutorial-array_stencil-doof2d-stencil_creation"></co>
+ !   Stencil&lt;DoofNinePt&gt; stencil;
+   
+     // Perform the simulation.
+ !   for (int k = 0; k &lt; nuIterations; ++k) {
+ !     // Read from b.  Write to a.  <co id="tutorial-array_stencil-doof2d-first_write"></co>
+       a(interiorDomain) = stencil(b, interiorDomain);
+   
+ ***************
+ *** 85,89 ****
+     // Print out the final central value.
+     Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+ !   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
+   
+     // The arrays are automatically deallocated.
+ --- 86,90 ----
+     // Print out the final central value.
+     Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+ !   std::cout &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
+   
+     // The arrays are automatically deallocated.
+ ***************
+ *** 93,94 ****
+ --- 94,96 ----
+     return EXIT_SUCCESS;
+   }
+ + </programlisting>
Index: docs/manual/programs/Doof2d-C-element-annotated.patch
===================================================================
RCS file: Doof2d-C-element-annotated.patch
diff -N Doof2d-C-element-annotated.patch
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-C-element-annotated.patch	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,150 ----
+ *** Doof2d-C-element.cpp	Tue Nov 27 08:36:38 2001
+ --- Doof2d-C-element-annotated.cpp	Tue Nov 27 12:08:03 2001
+ ***************
+ *** 1,4 ****
+ ! #include <iostream>		// has std::cout, ...
+ ! #include <stdlib.h>		// has EXIT_SUCCESS
+   
+   // Doof2d: C-like, element-wise implementation
+ --- 1,5 ----
+ ! <programlisting id="tutorial-hand_coded-doof2d-program" linenumbering="numbered" format="linespecific">
+ ! #include &lt;iostream&gt;		// has std::cout, ...
+ ! #include &lt;stdlib.h&gt;		// has EXIT_SUCCESS
+   
+   // Doof2d: C-like, element-wise implementation
+ ***************
+ *** 6,30 ****
+   int main()
+   {
+ !   // Ask the user for the number of averagings.
+     long nuAveragings, nuIterations;
+ !   std::cout << "Please enter the number of averagings: ";
+ !   std::cin >> nuAveragings;
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ !   // Use two-dimensional grids of values.
+     double **a;
+     double **b;
+   
+     // Ask the user for the number n of elements along one dimension of
+ !   // the grid.
+     long n;
+ !   std::cout << "Please enter the array size: ";
+ !   std::cin >> n;
+   
+ !   // Allocate the arrays.
+     typedef double* doublePtr;
+     a = new doublePtr[n];
+     b = new doublePtr[n];
+ !   for (int i = 0; i < n; i++) {
+       a[i] = new double[n];
+       b[i] = new double[n];
+ --- 7,31 ----
+   int main()
+   {
+ !   // Ask the user for the number of averagings.  <co id="tutorial-hand_coded-doof2d-nuaveragings"></co>
+     long nuAveragings, nuIterations;
+ !   std::cout &lt;&lt; "Please enter the number of averagings: ";
+ !   std::cin &gt;&gt; nuAveragings;
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ !   // Use two-dimensional grids of values.  <co id="tutorial-hand_coded-doof2d-array_storage"></co>
+     double **a;
+     double **b;
+   
+     // Ask the user for the number n of elements along one dimension of
+ !   // the grid.  <co id="tutorial-hand_coded-doof2d-grid_size"></co>
+     long n;
+ !   std::cout &lt;&lt; "Please enter the array size: ";
+ !   std::cin &gt;&gt; n;
+   
+ !   // Allocate the arrays.  <co id="tutorial-hand_coded-doof2d-allocation"></co>
+     typedef double* doublePtr;
+     a = new doublePtr[n];
+     b = new doublePtr[n];
+ !   for (int i = 0; i &lt; n; i++) {
+       a[i] = new double[n];
+       b[i] = new double[n];
+ ***************
+ *** 32,49 ****
+   
+     // Set up the initial conditions.
+ !   // All grid values should be zero except for the central value.
+ !   for (int j = 0; j < n; j++)
+ !     for (int i = 0; i < n; i++)
+         a[i][j] = b[i][j] = 0.0;
+     b[n/2][n/2] = 1000.0;
+   
+ !   // In the average, weight element with this value.
+     const double weight = 1.0/9.0;
+   
+     // Perform the simulation.
+ !   for (int k = 0; k < nuIterations; ++k) {
+ !     // Read from b.  Write to a.
+ !     for (int j = 1; j < n-1; j++)
+ !       for (int i = 1; i < n-1; i++)
+           a[i][j] = weight *
+             (b[i+1][j+1] + b[i+1][j  ] + b[i+1][j-1] +
+ --- 33,50 ----
+   
+     // Set up the initial conditions.
+ !   // All grid values should be zero except for the central value.  <co id="tutorial-hand_coded-doof2d-initialization"></co>
+ !   for (int j = 0; j &lt; n; j++)
+ !     for (int i = 0; i &lt; n; i++)
+         a[i][j] = b[i][j] = 0.0;
+     b[n/2][n/2] = 1000.0;
+   
+ !   // In the average, weight element with this value.  <co id="tutorial-hand_coded-doof2d-constants"></co>
+     const double weight = 1.0/9.0;
+   
+     // Perform the simulation.
+ !   for (int k = 0; k &lt; nuIterations; ++k) {
+ !     // Read from b.  Write to a.  <co id="tutorial-hand_coded-doof2d-first_write"></co>
+ !     for (int j = 1; j &lt; n-1; j++)
+ !       for (int i = 1; i &lt; n-1; i++)
+           a[i][j] = weight *
+             (b[i+1][j+1] + b[i+1][j  ] + b[i+1][j-1] +
+ ***************
+ *** 51,57 ****
+              b[i-1][j+1] + b[i-1][j  ] + b[i-1][j-1]);
+   
+ !     // Read from a.  Write to b.
+ !     for (int j = 1; j < n-1; j++)
+ !       for (int i = 1; i < n-1; i++)
+           b[i][j] = weight *
+             (a[i+1][j+1] + a[i+1][j  ] + a[i+1][j-1] +
+ --- 52,58 ----
+              b[i-1][j+1] + b[i-1][j  ] + b[i-1][j-1]);
+   
+ !     // Read from a.  Write to b.  <co id="tutorial-hand_coded-doof2d-second_write"></co>
+ !     for (int j = 1; j &lt; n-1; j++)
+ !       for (int i = 1; i &lt; n-1; i++)
+           b[i][j] = weight *
+             (a[i+1][j+1] + a[i+1][j  ] + a[i+1][j-1] +
+ ***************
+ *** 60,68 ****
+     }
+   
+ !   // Print out the final central value.
+ !   std::cout << (nuAveragings % 2 ? a[n/2][n/2] : b[n/2][n/2]) << std::endl;
+   
+ !   // Deallocate the arrays.
+ !   for (int i = 0; i < n; i++) {
+       delete [] a[i];
+       delete [] b[i];
+ --- 61,69 ----
+     }
+   
+ !   // Print out the final central value.  <co id="tutorial-hand_coded-doof2d-answer"></co>
+ !   std::cout &lt;&lt; (nuAveragings % 2 ? a[n/2][n/2] : b[n/2][n/2]) &lt;&lt; std::endl;
+   
+ !   // Deallocate the arrays.  <co id="tutorial-hand_coded-doof2d-deallocation"></co>
+ !   for (int i = 0; i &lt; n; i++) {
+       delete [] a[i];
+       delete [] b[i];
+ ***************
+ *** 73,74 ****
+ --- 74,76 ----
+     return EXIT_SUCCESS;
+   }
+ + </programlisting>
Index: docs/manual/programs/makefile
===================================================================
RCS file: makefile
diff -N makefile
*** /dev/null	Fri Mar 23 21:37:44 2001
--- makefile	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,12 ----
+ ### Oldham, Jeffrey D.
+ ### 2001Nov27
+ ### Pooma
+ ###
+ ### Produce Annotated Source Code
+ 
+ all: Doof2d-C-element-annotated.cpp Doof2d-Array-element-annotated.cpp \
+      Doof2d-Array-parallel-annotated.cpp Doof2d-Array-stencil-annotated.cpp \
+      Doof2d-Array-distributed-annotated.cpp
+ 
+ %-annotated.cpp: %-annotated.patch %.cpp
+ 	patch -o $@ < $<
Index: examples/Manual/Doof2d/Doof2d-Array-distributed.cpp
===================================================================
RCS file: Doof2d-Array-distributed.cpp
diff -N Doof2d-Array-distributed.cpp
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-Array-distributed.cpp	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,111 ----
+ #include <iostream>		// has std::cout, ...
+ #include <stdlib.h>		// has EXIT_SUCCESS
+ #include "Pooma/Arrays.h"	// has Pooma's Array
+ 
+ // Doof2d: Pooma Arrays, stencil, multiple processor implementation
+ 
+ // Define the stencil class performing the computation.
+ class DoofNinePt
+ {
+ public:
+   // Initialize the constant average weighting.
+   DoofNinePt() : weight(1.0/9.0) {}
+ 
+   // This stencil operator is applied to each interior domain position
+   // (i,j).  The "C" template parameter permits use of this stencil
+   // operator with both Arrays and Fields.
+   template <class C>
+   inline
+   typename C::Element_t
+   operator()(const C& x, int i, int j) const
+   {
+     return ( weight *
+              ( x.read(i+1,j+1) + x.read(i+1,j  ) + x.read(i+1,j-1) +
+                x.read(i  ,j+1) + x.read(i  ,j  ) + x.read(i  ,j-1) +
+                x.read(i-1,j+1) + x.read(i-1,j  ) + x.read(i-1,j-1) ) );
+   }
+ 
+   inline int lowerExtent(int) const { return 1; }
+   inline int upperExtent(int) const { return 1; }
+ 
+ private:
+ 
+   // In the average, weight element with this value.
+   const double weight;
+ };
+ 
+ int main(int argc, char *argv[])
+ {
+   // Prepare the Pooma library for execution.
+   Pooma::initialize(argc,argv);
+   
+   // Ask the user for the number of processors.
+   long nuProcessors;
+   std::cout << "Please enter the number of processors: ";
+   std::cin >> nuProcessors;
+ 
+   // Ask the user for the number of averagings.
+   long nuAveragings, nuIterations;
+   std::cout << "Please enter the number of averagings: ";
+   std::cin >> nuAveragings;
+   nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+ 
+   // Ask the user for the number n of elements along one dimension of
+   // the grid.
+   long n;
+   std::cout << "Please enter the array size: ";
+   std::cin >> n;
+ 
+   // Specify the arrays' domains [0,n) x [0,n).
+   Interval<1> N(0, n-1);
+   Interval<2> vertDomain(N, N);
+ 
+   // Set up interior domains [1,n-1) x [1,n-1) for computation.
+   Interval<1> I(1,n-2);
+   Interval<2> interiorDomain(I,I);
+ 
+   // Create the distributed arrays.
+   // Partition the arrays' domains uniformly, i.e., each patch has the
+   // same size.  The first parameter tells how many patches for each
+   // dimension.  Guard layers optimize communication between patches.
+   // Internal guards surround each patch.  External guards surround
+   // the entire array domain.
+   UniformGridPartition<2> partition(Loc<2>(nuProcessors, nuProcessors),
+ 				    GuardLayers<2>(1),  // internal
+ 				    GuardLayers<2>(0)); // external
+   UniformGridLayout<2> layout(vertDomain, partition, DistributedTag());
+ 
+   // The template parameters indicate 2 dimensions and a 'double'
+   // element type.  MultiPatch indicates multiple computation patches,
+   // i.e., distributed computation.  The UniformTag indicates the
+   // patches should have the same size.  Each patch has Brick type.
+   Array<2, double, MultiPatch<UniformTag, Remote<Brick> > > a(layout);
+   Array<2, double, MultiPatch<UniformTag, Remote<Brick> > > b(layout);
+ 
+   // Set up the initial conditions.
+   // All grid values should be zero except for the central value.
+   a = b = 0.0;
+   b(n/2,n/2) = 1000.0;
+ 
+   // Create the stencil performing the computation.
+   Stencil<DoofNinePt> stencil;
+ 
+   // Perform the simulation.
+   for (int k = 0; k < nuIterations; ++k) {
+     // Read from b.  Write to a.
+     a(interiorDomain) = stencil(b, interiorDomain);
+ 
+     // Read from a.  Write to b.
+     b(interiorDomain) = stencil(a, interiorDomain);
+   }
+ 
+   // Print out the final central value.
+   Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
+ 
+   // The arrays are automatically deallocated.
+ 
+   // Tell the Pooma library execution has finished.
+   Pooma::finalize();
+   return EXIT_SUCCESS;
+ }
Index: examples/Manual/Doof2d/Doof2d-Array-element.cpp
===================================================================
RCS file: Doof2d-Array-element.cpp
diff -N Doof2d-Array-element.cpp
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-Array-element.cpp	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,70 ----
+ #include <iostream>		// has std::cout, ...
+ #include <stdlib.h>		// has EXIT_SUCCESS
+ #include "Pooma/Arrays.h"	// has Pooma's Array
+ 
+ // Doof2d: Pooma Arrays, element-wise implementation
+ 
+ int main(int argc, char *argv[])
+ {
+   // Prepare the Pooma library for execution.
+   Pooma::initialize(argc,argv);
+   
+   // Ask the user for the number of averagings.
+   long nuAveragings, nuIterations;
+   std::cout << "Please enter the number of averagings: ";
+   std::cin >> nuAveragings;
+   nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+ 
+   // Ask the user for the number n of elements along one dimension of
+   // the grid.
+   long n;
+   std::cout << "Please enter the array size: ";
+   std::cin >> n;
+ 
+   // Specify the arrays' domains [0,n) x [0,n).
+   Interval<1> N(0, n-1);
+   Interval<2> vertDomain(N, N);
+ 
+   // Create the arrays.
+   // The template parameters indicate 2 dimensions, a 'double' element
+   // type, and ordinary 'Brick' storage.
+   Array<2, double, Brick> a(vertDomain);
+   Array<2, double, Brick> b(vertDomain);
+ 
+   // Set up the initial conditions.
+   // All grid values should be zero except for the central value.
+   a = b = 0.0;
+   b(n/2,n/2) = 1000.0;
+ 
+   // In the average, weight element with this value.
+   const double weight = 1.0/9.0;
+ 
+   // Perform the simulation.
+   for (int k = 0; k < nuIterations; ++k) {
+     // Read from b.  Write to a.
+     for (int j = 1; j < n-1; j++)
+       for (int i = 1; i < n-1; i++)
+         a(i,j) = weight *
+           (b(i+1,j+1) + b(i+1,j  ) + b(i+1,j-1) +
+            b(i  ,j+1) + b(i  ,j  ) + b(i  ,j-1) +
+            b(i-1,j+1) + b(i-1,j  ) + b(i-1,j-1));
+ 
+     // Read from a.  Write to b.
+     for (int j = 1; j < n-1; j++)
+       for (int i = 1; i < n-1; i++)
+         b(i,j) = weight *
+           (a(i+1,j+1) + a(i+1,j  ) + a(i+1,j-1) +
+            a(i  ,j+1) + a(i  ,j  ) + a(i  ,j-1) +
+            a(i-1,j+1) + a(i-1,j  ) + a(i-1,j-1));
+   }
+ 
+   // Print out the final central value.
+   Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
+ 
+   // The arrays are automatically deallocated.
+ 
+   // Tell the Pooma library execution has finished.
+   Pooma::finalize();
+   return EXIT_SUCCESS;
+ }
Index: examples/Manual/Doof2d/Doof2d-Array-parallel.cpp
===================================================================
RCS file: Doof2d-Array-parallel.cpp
diff -N Doof2d-Array-parallel.cpp
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-Array-parallel.cpp	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,70 ----
+ #include <iostream>		// has std::cout, ...
+ #include <stdlib.h>		// has EXIT_SUCCESS
+ #include "Pooma/Arrays.h"	// has Pooma's Array
+ 
+ // Doof2d: Pooma Arrays, data-parallel implementation
+ 
+ int main(int argc, char *argv[])
+ {
+   // Prepare the Pooma library for execution.
+   Pooma::initialize(argc,argv);
+   
+   // Ask the user for the number of averagings.
+   long nuAveragings, nuIterations;
+   std::cout << "Please enter the number of averagings: ";
+   std::cin >> nuAveragings;
+   nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+ 
+   // Ask the user for the number n of elements along one dimension of
+   // the grid.
+   long n;
+   std::cout << "Please enter the array size: ";
+   std::cin >> n;
+ 
+   // Specify the arrays' domains [0,n) x [0,n).
+   Interval<1> N(0, n-1);
+   Interval<2> vertDomain(N, N);
+ 
+   // Set up interior domains [1,n-1) x [1,n-1) for computation.
+   Interval<1> I(1,n-2);
+   Interval<1> J(1,n-2);
+ 
+   // Create the arrays.
+   // The template parameters indicate 2 dimensions, a 'double' element
+   // type, and ordinary 'Brick' storage.
+   Array<2, double, Brick> a(vertDomain);
+   Array<2, double, Brick> b(vertDomain);
+ 
+   // Set up the initial conditions.
+   // All grid values should be zero except for the central value.
+   a = b = 0.0;
+   b(n/2,n/2) = 1000.0;
+ 
+   // In the average, weight element with this value.
+   const double weight = 1.0/9.0;
+ 
+   // Perform the simulation.
+   for (int k = 0; k < nuIterations; ++k) {
+     // Read from b.  Write to a.
+     a(I,J) = weight *
+       (b(I+1,J+1) + b(I+1,J  ) + b(I+1,J-1) +
+        b(I  ,J+1) + b(I  ,J  ) + b(I  ,J-1) +
+        b(I-1,J+1) + b(I-1,J  ) + b(I-1,J-1));
+ 
+     // Read from a.  Write to b.
+     b(I,J) = weight *
+       (a(I+1,J+1) + a(I+1,J  ) + a(I+1,J-1) +
+        a(I  ,J+1) + a(I  ,J  ) + a(I  ,J-1) +
+        a(I-1,J+1) + a(I-1,J  ) + a(I-1,J-1));
+   }
+ 
+   // Print out the final central value.
+   Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
+ 
+   // The arrays are automatically deallocated.
+ 
+   // Tell the Pooma library execution has finished.
+   Pooma::finalize();
+   return EXIT_SUCCESS;
+ }
Index: examples/Manual/Doof2d/Doof2d-Array-stencil.cpp
===================================================================
RCS file: Doof2d-Array-stencil.cpp
diff -N Doof2d-Array-stencil.cpp
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-Array-stencil.cpp	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,94 ----
+ #include <iostream>		// has std::cout, ...
+ #include <stdlib.h>		// has EXIT_SUCCESS
+ #include "Pooma/Arrays.h"	// has Pooma's Array
+ 
+ // Doof2d: Pooma Arrays, stencil implementation
+ 
+ // Define the stencil class performing the computation.
+ class DoofNinePt
+ {
+ public:
+   // Initialize the constant average weighting.
+   DoofNinePt() : weight(1.0/9.0) {}
+ 
+   // This stencil operator is applied to each interior domain position
+   // (i,j).  The "C" template parameter permits use of this stencil
+   // operator with both Arrays and Fields.
+   template <class C>
+   inline
+   typename C::Element_t
+   operator()(const C& c, int i, int j) const
+   {
+     return ( weight *
+              ( c.read(i+1,j+1) + c.read(i+1,j  ) + c.read(i+1,j-1) +
+                c.read(i  ,j+1) + c.read(i  ,j  ) + c.read(i  ,j-1) +
+                c.read(i-1,j+1) + c.read(i-1,j  ) + c.read(i-1,j-1) ) );
+   }
+ 
+   inline int lowerExtent(int) const { return 1; }
+   inline int upperExtent(int) const { return 1; }
+ 
+ private:
+ 
+   // In the average, weight element with this value.
+   const double weight;
+ };
+ 
+ int main(int argc, char *argv[])
+ {
+   // Prepare the Pooma library for execution.
+   Pooma::initialize(argc,argv);
+   
+   // Ask the user for the number of averagings.
+   long nuAveragings, nuIterations;
+   std::cout << "Please enter the number of averagings: ";
+   std::cin >> nuAveragings;
+   nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+ 
+   // Ask the user for the number n of elements along one dimension of
+   // the grid.
+   long n;
+   std::cout << "Please enter the array size: ";
+   std::cin >> n;
+ 
+   // Specify the arrays' domains [0,n) x [0,n).
+   Interval<1> N(0, n-1);
+   Interval<2> vertDomain(N, N);
+ 
+   // Set up interior domains [1,n-1) x [1,n-1) for computation.
+   Interval<1> I(1,n-2);
+   Interval<2> interiorDomain(I,I);
+ 
+   // Create the arrays.
+   // The template parameters indicate 2 dimensions, a 'double' element
+   // type, and ordinary 'Brick' storage.
+   Array<2, double, Brick> a(vertDomain);
+   Array<2, double, Brick> b(vertDomain);
+ 
+   // Set up the initial conditions.
+   // All grid values should be zero except for the central value.
+   a = b = 0.0;
+   b(n/2,n/2) = 1000.0;
+ 
+   // Create the stencil performing the computation.
+   Stencil<DoofNinePt> stencil;
+ 
+   // Perform the simulation.
+   for (int k = 0; k < nuIterations; ++k) {
+     // Read from b.  Write to a.
+     a(interiorDomain) = stencil(b, interiorDomain);
+ 
+     // Read from a.  Write to b.
+     b(interiorDomain) = stencil(a, interiorDomain);
+   }
+ 
+   // Print out the final central value.
+   Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
+ 
+   // The arrays are automatically deallocated.
+ 
+   // Tell the Pooma library execution has finished.
+   Pooma::finalize();
+   return EXIT_SUCCESS;
+ }
Index: examples/Manual/Doof2d/Doof2d-C-element.cpp
===================================================================
RCS file: Doof2d-C-element.cpp
diff -N Doof2d-C-element.cpp
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-C-element.cpp	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,74 ----
+ #include <iostream>		// has std::cout, ...
+ #include <stdlib.h>		// has EXIT_SUCCESS
+ 
+ // Doof2d: C-like, element-wise implementation
+ 
+ int main()
+ {
+   // Ask the user for the number of averagings.
+   long nuAveragings, nuIterations;
+   std::cout << "Please enter the number of averagings: ";
+   std::cin >> nuAveragings;
+   nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+ 
+   // Use two-dimensional grids of values.
+   double **a;
+   double **b;
+ 
+   // Ask the user for the number n of elements along one dimension of
+   // the grid.
+   long n;
+   std::cout << "Please enter the array size: ";
+   std::cin >> n;
+ 
+   // Allocate the arrays.
+   typedef double* doublePtr;
+   a = new doublePtr[n];
+   b = new doublePtr[n];
+   for (int i = 0; i < n; i++) {
+     a[i] = new double[n];
+     b[i] = new double[n];
+   }
+ 
+   // Set up the initial conditions.
+   // All grid values should be zero except for the central value.
+   for (int j = 0; j < n; j++)
+     for (int i = 0; i < n; i++)
+       a[i][j] = b[i][j] = 0.0;
+   b[n/2][n/2] = 1000.0;
+ 
+   // In the average, weight element with this value.
+   const double weight = 1.0/9.0;
+ 
+   // Perform the simulation.
+   for (int k = 0; k < nuIterations; ++k) {
+     // Read from b.  Write to a.
+     for (int j = 1; j < n-1; j++)
+       for (int i = 1; i < n-1; i++)
+         a[i][j] = weight *
+           (b[i+1][j+1] + b[i+1][j  ] + b[i+1][j-1] +
+            b[i  ][j+1] + b[i  ][j  ] + b[i  ][j-1] +
+            b[i-1][j+1] + b[i-1][j  ] + b[i-1][j-1]);
+ 
+     // Read from a.  Write to b.
+     for (int j = 1; j < n-1; j++)
+       for (int i = 1; i < n-1; i++)
+         b[i][j] = weight *
+           (a[i+1][j+1] + a[i+1][j  ] + a[i+1][j-1] +
+            a[i  ][j+1] + a[i  ][j  ] + a[i  ][j-1] +
+            a[i-1][j+1] + a[i-1][j  ] + a[i-1][j-1]);
+   }
+ 
+   // Print out the final central value.
+   std::cout << (nuAveragings % 2 ? a[n/2][n/2] : b[n/2][n/2]) << std::endl;
+ 
+   // Deallocate the arrays.
+   for (int i = 0; i < n; i++) {
+     delete [] a[i];
+     delete [] b[i];
+   }
+   delete [] a;
+   delete [] b;
+ 
+   return EXIT_SUCCESS;
+ }
Index: examples/Manual/Doof2d/Doof2d-Field-distributed.cpp
===================================================================
RCS file: Doof2d-Field-distributed.cpp
diff -N Doof2d-Field-distributed.cpp
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-Field-distributed.cpp	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,87 ----
+ #include <iostream>		// has std::cout, ...
+ #include <stdlib.h>		// has EXIT_SUCCESS
+ #include "Pooma/Fields.h"	// has Pooma's Field
+ 
+ // Doof2d: Pooma Fields, data-parallel, multiple processor implementation
+ 
+ int main(int argc, char *argv[])
+ {
+   // Prepare the Pooma library for execution.
+   Pooma::initialize(argc,argv);
+   
+   // nuIterations is the number of simulation iterations.
+   const int nuIterations = 10/2;
+ 
+   // In the average, weight element with this value.
+   const double weight = 1.0/9.0;
+ 
+   // nuProcessors is the number of processors along one dimension.
+   const int nuProcessors = 2;
+ 
+   // Ask the user for the number n of elements along one dimension of
+   // the grid.
+   long n;
+   std::cout << "Please enter the array size: ";
+   std::cin >> n;
+ 
+   // Specify the fields' domains [0,n) x [0,n).
+   Interval<1> N(0, n-1);
+   Interval<2> vertDomain(N, N);
+ 
+   // Set up interior domains [1,n-1) x [1,n-1) for computation.
+   Interval<1> I(1,n-2);
+   Interval<1> J(1,n-2);
+ 
+   // Partition the fields' domains uniformly, i.e., each patch has the
+   // same size.  The first parameter tells how many patches for each
+   // dimension.  Guard layers optimize communication between patches.
+   // Internal guards surround each patch.  External guards surround
+   // the entire field domain.
+   UniformGridPartition<2> partition(Loc<2>(nuProcessors, nuProcessors),
+ 				    GuardLayers<2>(1),  // internal
+ 				    GuardLayers<2>(0)); // external
+   UniformGridLayout<2> layout(vertDomain, partition, DistributedTag());
+ 
+   // Specify the fields' mesh, i.e., its spatial extent, and its
+   // centering type.
+   UniformRectilinearMesh<2> mesh(layout, Vector<2>(0.0), Vector<2>(1.0, 1.0));
+   Centering<2> cell = canonicalCentering<2>(CellType, Continuous, AllDim);
+ 
+   // The template parameters indicate a mesh and a 'double'
+   // element type.  MultiPatch indicates multiple computation patches,
+   // i.e., distributed computation.  The UniformTag indicates the
+   // patches should have the same size.  Each patch has Brick type.
+   Field<UniformRectilinearMesh<2>, double, MultiPatch<UniformTag,
+     Remote<Brick> > > a(cell, layout, mesh);
+   Field<UniformRectilinearMesh<2>, double, MultiPatch<UniformTag,
+     Remote<Brick> > > b(cell, layout, mesh);
+ 
+   // Set up the initial conditions.
+   // All grid values should be zero except for the central value.
+   a = b = 0.0;
+   b(n/2,n/2) = 1000.0;
+ 
+   // Perform the simulation.
+   for (int k = 0; k < nuIterations; ++k) {
+     // Read from b.  Write to a.
+     a(I,J) = weight *
+       (b(I+1,J+1) + b(I+1,J  ) + b(I+1,J-1) +
+        b(I  ,J+1) + b(I  ,J  ) + b(I  ,J-1) +
+        b(I-1,J+1) + b(I-1,J  ) + b(I-1,J-1));
+ 
+     // Read from a.  Write to b.
+     b(I,J) = weight *
+       (a(I+1,J+1) + a(I+1,J  ) + a(I+1,J-1) +
+        a(I  ,J+1) + a(I  ,J  ) + a(I  ,J-1) +
+        a(I-1,J+1) + a(I-1,J  ) + a(I-1,J-1));
+   }
+ 
+   // Print out the final central value.
+   std::cout << b(n/2,n/2) << std::endl;
+ 
+   // The fields are automatically deallocated.
+ 
+   // Tell the Pooma library execution has finished.
+   Pooma::finalize();
+   return EXIT_SUCCESS;
+ }
Index: examples/Manual/Doof2d/Doof2d-Field-parallel.cpp
===================================================================
RCS file: Doof2d-Field-parallel.cpp
diff -N Doof2d-Field-parallel.cpp
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-Field-parallel.cpp	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,72 ----
+ #include <iostream>		// has std::cout, ...
+ #include <stdlib.h>		// has EXIT_SUCCESS
+ #include "Pooma/Fields.h"	// has Pooma's Field
+ 
+ // Doof2d: Pooma Fields, data-parallel implementation
+ 
+ int main(int argc, char *argv[])
+ {
+   // Prepare the Pooma library for execution.
+   Pooma::initialize(argc,argv);
+   
+   // nuIterations is the number of simulation iterations.
+   const int nuIterations = 10/2;
+ 
+   // In the average, weight element with this value.
+   const double weight = 1.0/9.0;
+ 
+   // Ask the user for the number n of elements along one dimension of
+   // the grid.
+   long n;
+   std::cout << "Please enter the array size: ";
+   std::cin >> n;
+ 
+   // Specify the fields' domains [0,n) x [0,n).
+   Interval<1> N(0, n-1);
+   Interval<2> vertDomain(N, N);
+ 
+   // Set up interior domains [1,n-1) x [1,n-1) for computation.
+   Interval<1> I(1,n-2);
+   Interval<1> J(1,n-2);
+ 
+   // Specify the fields' mesh, i.e., its spatial extent, and its
+   // centering type.
+   DomainLayout<2> layout(vertDomain);
+   UniformRectilinearMesh<2> mesh(layout, Vector<2>(0.0), Vector<2>(1.0, 1.0));
+   Centering<2> cell = canonicalCentering<2>(CellType, Continuous, AllDim);
+ 
+   // Create the fields.
+   // The template parameters indicate a mesh, a 'double' element
+   // type, and ordinary 'Brick' storage.
+   Field<UniformRectilinearMesh<2>, double, Brick> a(cell, layout, mesh);
+   Field<UniformRectilinearMesh<2>, double, Brick> b(cell, layout, mesh);
+ 
+   // Set up the initial conditions.
+   // All grid values should be zero except for the central value.
+   a = b = 0.0;
+   b(n/2,n/2) = 1000.0;
+ 
+   // Perform the simulation.
+   for (int k = 0; k < nuIterations; ++k) {
+     // Read from b.  Write to a.
+     a(I,J) = weight *
+       (b(I+1,J+1) + b(I+1,J  ) + b(I+1,J-1) +
+        b(I  ,J+1) + b(I  ,J  ) + b(I  ,J-1) +
+        b(I-1,J+1) + b(I-1,J  ) + b(I-1,J-1));
+ 
+     // Read from a.  Write to b.
+     b(I,J) = weight *
+       (a(I+1,J+1) + a(I+1,J  ) + a(I+1,J-1) +
+        a(I  ,J+1) + a(I  ,J  ) + a(I  ,J-1) +
+        a(I-1,J+1) + a(I-1,J  ) + a(I-1,J-1));
+   }
+ 
+   // Print out the final central value.
+   std::cout << b(n/2,n/2) << std::endl;
+ 
+   // The fields are automatically deallocated.
+ 
+   // Tell the Pooma library execution has finished.
+   Pooma::finalize();
+   return EXIT_SUCCESS;
+ }
Index: examples/Manual/Doof2d/include.mk
===================================================================
RCS file: include.mk
diff -N include.mk
*** /dev/null	Fri Mar 23 21:37:44 2001
--- include.mk	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,59 ----
+ # Generated by mm.pl: Mon Mar  9 13:58:39 MST 1998
+ # ACL:license
+ #  ----------------------------------------------------------------------
+ #  This software and ancillary information (herein called "SOFTWARE")
+ #  called POOMA (Parallel Object-Oriented Methods and Applications) is
+ #  made available under the terms described here.  The SOFTWARE has been
+ #  approved for release with associated LA-CC Number LA-CC-98-65.
+ #  
+ #  Unless otherwise indicated, this SOFTWARE has been authored by an
+ #  employee or employees of the University of California, operator of the
+ #  Los Alamos National Laboratory under Contract No. W-7405-ENG-36 with
+ #  the U.S. Department of Energy.  The U.S. Government has rights to use,
+ #  reproduce, and distribute this SOFTWARE. The public may copy, distribute,
+ #  prepare derivative works and publicly display this SOFTWARE without 
+ #  charge, provided that this Notice and any statement of authorship are 
+ #  reproduced on all copies.  Neither the Government nor the University 
+ #  makes any warranty, express or implied, or assumes any liability or 
+ #  responsibility for the use of this SOFTWARE.
+ #  
+ #  If SOFTWARE is modified to produce derivative works, such modified
+ #  SOFTWARE should be clearly marked, so as not to confuse it with the
+ #  version available from LANL.
+ #  
+ #  For more information about POOMA, send e-mail to pooma at acl.lanl.gov,
+ #  or visit the POOMA web page at http://www.acl.lanl.gov/pooma/.
+ #  ----------------------------------------------------------------------
+ # ACL:license
+ 
+ 
+ # Wrap make components from SHARED_ROOT and the current directory in the 
+ # proper order so that variables like ODIR have the correct directory-specific 
+ # value at the right moment.  See the included files for details of what they 
+ # are doing. This file should NOT be manually edited.
+ 
+ # Set NEXTDIR, THISDIR and DIR_LIST
+ include $(SHARED_ROOT)/include1.mk
+ 
+ # Include list of subdirectories to process
+ -include $(THISDIR)/subdir.mk
+ 
+ # Set ODIR, PROJECT_INCLUDES, UNIQUE
+ include $(SHARED_ROOT)/include2.mk
+ 
+ # Set list of object files, relative to ODIR
+ -include $(THISDIR)/objfile.mk
+ 
+ # Set rules for the ODIR directory 
+ include $(SHARED_ROOT)/compilerules.mk
+ 
+ # Remove current dir from DIR_LIST
+ DIR_LIST :=$(filter-out $(firstword $(DIR_LIST)), $(DIR_LIST))
+ 
+ 
+ # ACL:rcsinfo
+ #  ----------------------------------------------------------------------
+ #  $RCSfile: include.mk,v $   $Author: swhaney $
+ #  $Revision: 1.3 $   $Date: 2000/03/07 13:14:47 $
+ #  ----------------------------------------------------------------------
+ # ACL:rcsinfo
Index: examples/Manual/Doof2d/makefile
===================================================================
RCS file: makefile
diff -N makefile
*** /dev/null	Fri Mar 23 21:37:44 2001
--- makefile	Mon Dec  3 14:01:55 2001
***************
*** 0 ****
--- 1,96 ----
+ # Generated by mm.pl: Mon Mar  9 13:58:39 MST 1998
+ # ACL:license
+ #  ----------------------------------------------------------------------
+ #  This software and ancillary information (herein called "SOFTWARE")
+ #  called POOMA (Parallel Object-Oriented Methods and Applications) is
+ #  made available under the terms described here.  The SOFTWARE has been
+ #  approved for release with associated LA-CC Number LA-CC-98-65.
+ #  
+ #  Unless otherwise indicated, this SOFTWARE has been authored by an
+ #  employee or employees of the University of California, operator of the
+ #  Los Alamos National Laboratory under Contract No. W-7405-ENG-36 with
+ #  the U.S. Department of Energy.  The U.S. Government has rights to use,
+ #  reproduce, and distribute this SOFTWARE. The public may copy, distribute,
+ #  prepare derivative works and publicly display this SOFTWARE without 
+ #  charge, provided that this Notice and any statement of authorship are 
+ #  reproduced on all copies.  Neither the Government nor the University 
+ #  makes any warranty, express or implied, or assumes any liability or 
+ #  responsibility for the use of this SOFTWARE.
+ #  
+ #  If SOFTWARE is modified to produce derivative works, such modified
+ #  SOFTWARE should be clearly marked, so as not to confuse it with the
+ #  version available from LANL.
+ #  
+ #  For more information about POOMA, send e-mail to pooma at acl.lanl.gov,
+ #  or visit the POOMA web page at http://www.acl.lanl.gov/pooma/.
+ #  ----------------------------------------------------------------------
+ # ACL:license
+ 
+ # This file is user-editable
+ 
+ PROJECT_ROOT = $(shell cd ../../..; pwd)
+ include $(PROJECT_ROOT)/config/head.mk
+ 
+ PASS=APP
+ 
+ default:: Doof2d-C-element Doof2d-Array-element Doof2d-Array-parallel \
+ 	Doof2d-Array-stencil Doof2d-Array-distributed \
+ 	Doof2d-Field-parallel Doof2d-Field-distributed
+ 
+ .PHONY: Doof2d-C-element
+ 
+ Doof2d-C-element:: $(ODIR)/Doof2d-C-element
+ 
+ $(ODIR)/Doof2d-C-element: $(ODIR)/Doof2d-C-element.o
+ 	$(LinkToSuite)
+ 
+ .PHONY: Doof2d-Array-element
+ 
+ Doof2d-Array-element:: $(ODIR)/Doof2d-Array-element
+ 
+ $(ODIR)/Doof2d-Array-element: $(ODIR)/Doof2d-Array-element.o
+ 	$(LinkToSuite)
+ 
+ .PHONY: Doof2d-Array-parallel
+ 
+ Doof2d-Array-parallel:: $(ODIR)/Doof2d-Array-parallel
+ 
+ $(ODIR)/Doof2d-Array-parallel: $(ODIR)/Doof2d-Array-parallel.o
+ 	$(LinkToSuite)
+ 
+ .PHONY: Doof2d-Array-stencil
+ 
+ Doof2d-Array-stencil:: $(ODIR)/Doof2d-Array-stencil
+ 
+ $(ODIR)/Doof2d-Array-stencil: $(ODIR)/Doof2d-Array-stencil.o
+ 	$(LinkToSuite)
+ 
+ .PHONY: Doof2d-Array-distributed
+ 
+ Doof2d-Array-distributed:: $(ODIR)/Doof2d-Array-distributed
+ 
+ $(ODIR)/Doof2d-Array-distributed: $(ODIR)/Doof2d-Array-distributed.o
+ 	$(LinkToSuite)
+ 
+ .PHONY: Doof2d-Field-parallel
+ 
+ Doof2d-Field-parallel:: $(ODIR)/Doof2d-Field-parallel
+ 
+ $(ODIR)/Doof2d-Field-parallel: $(ODIR)/Doof2d-Field-parallel.o
+ 	$(LinkToSuite)
+ 
+ .PHONY: Doof2d-Field-distributed
+ 
+ Doof2d-Field-distributed:: $(ODIR)/Doof2d-Field-distributed
+ 
+ $(ODIR)/Doof2d-Field-distributed: $(ODIR)/Doof2d-Field-distributed.o
+ 	$(LinkToSuite)
+ 
+ include $(SHARED_ROOT)/tail.mk
+ 
+ # ACL:rcsinfo
+ #  ----------------------------------------------------------------------
+ #  $RCSfile: makefile,v $   $Author: oldham $
+ #  $Revision: 1.1 $   $Date: 2000/07/21 21:34:44 $
+ #  ----------------------------------------------------------------------
+ # ACL:rcsinfo

From Bert.Tijskens at agr.kuleuven.ac.be  Tue Dec  4 07:10:39 2001
From: Bert.Tijskens at agr.kuleuven.ac.be (Tijskens, Bert)
Date: Tue, 4 Dec 2001 08:10:39 +0100
Subject: pooma performance
Message-ID: <EDBF7BA6F85D5A498FC04614033A24250224BA@AGR-SRV-17.agr.ad10.intern.kuleuven.ac.be>

Hi,
looking for support for POOMA++ 2.3.0 I found this e-mail adress
somewhere on the internet.
I wrote the following benchmark program which computes something of the
form y=3Dax+b, where y and x are (DynamicArrays of) vectors and a and b
are (DynamicArrays of) scalars. I was surprised and dissappointed to
find out that the simple c version of this loop is 4-5 times faster than
the POOMA version? I suppose I must have overlooked something. Can you
help? the program is at the bottom of this message

the tests were run on a PC using the Intel C++ compiler

many thanks in advance,
bert



Dr. Engelbert TIJSKENS

Laboratory for Agro-Machinery and -Processing
Department of Agro-Engineering and -Economy
KULeuven

Kasteelpark Arenberg 30
B-3001 LEUVEN
BELGIUM

tel: ++(32) 16 32 8557
fax: ++(32) 16 32 8590
e-mail: engelbert.tijskens at agr.kuleuven.ac.be



Here's the program

#include "Pooma/Particles.h"
#include "Pooma/DynamicArrays.h"
#include "Tiny/Vector.h"
#include "Utilities/Inform.h"
#include <iostream>
#include <stdlib.h>
#include <Timer/Timer.h>

#if POOMA_CHEETAH
	typedef MultiPatch< DynamicTag, Remote<Dynamic> >
AttributeEngineTag_t;
#else
	typedef MultiPatch< DynamicTag,        Dynamic  >
AttributeEngineTag_t;
#endif

template <class Layout_t>
struct PC_UniformLayout_traits
{
	typedef AttributeEngineTag_t AttributeEngineTag_t;
	typedef Layout_t ParticleLayout_t;
};


// The particle traits class and layout type for this application
typedef PC_UniformLayout_traits<UniformLayout> PC_UniformLayout_t;

// Dimensionality of this problem
static const int nsd =3D 3;
static const int NumPart =3D 10000;	// Number of particles in
simulation
static const int nLoops  =3D 100;		// Number of loops

// Particles subclass with position and velocity
class PC : public Particles<PC_UniformLayout_t>
{
public:
 // Typedefs
	typedef Particles<PC_UniformLayout_t>   Base_t;
	typedef Base_t::AttributeEngineTag_t	AttributeEngineTag_t;
	typedef Base_t::ParticleLayout_t
ParticleLayout_t;
	typedef double                          AxisType_t;
	typedef Vector<nsd,AxisType_t>         PointType_t;

 // Constructor: set up layouts, register attributes
	PC(const ParticleLayout_t &pl) :
Particles<PC_UniformLayout_t>(pl)
	{
		addAttribute(y);
		addAttribute(x);
		addAttribute(a);
		addAttribute(b);=09
	}

 // Position and velocity attributes (as public members)
	DynamicArray<PointType_t,AttributeEngineTag_t>  x,y;
	DynamicArray<AxisType_t ,AttributeEngineTag_t>  a,b;

	double x_[NumPart][nsd], y_[NumPart][nsd];
	double a_[NumPart], b_[NumPart];
};


// Number of patches to distribute particles across.
// Typically one would use one patch per processor.
const int numPatches =3D 1;


// Main simulation routine
int main(int argc, char *argv[])
{
 // Initialize POOMA and output stream
	Pooma::initialize(argc,argv);
	Inform out(argv[0]);
=09
	out << "Begin Bounce example code" << std::endl;
	out << "-------------------------" << std::endl;

 // Create a particle layout object for our use
	PC_UniformLayout_t::ParticleLayout_t particleLayout(numPatches);

 // Create the Particles subclass object
	PC pc(particleLayout);

 // Create some particles, recompute the global domain, and initialize
 // the attributes randomly.
	pc.globalCreate(NumPart);

	srand(12345U);
	typedef PC::AxisType_t Coordinate_t;
	Coordinate_t recranmax =3D
1.0/static_cast<Coordinate_t>(RAND_MAX);
	for (int i =3D 0; i < NumPart; ++i)
	{
		for (int d =3D 0; d < nsd; ++d) {
			pc.x(i)(d) =3D rand() * recranmax;
			pc.x_[i][d] =3D pc.x(i)(d);
		}
		pc.a_[i] =3D pc.a(i) =3D rand() * recranmax;
		pc.b_[i] =3D pc.b(i) =3D rand() * recranmax;
	}

 // reference using ordinary arrays : y =3D ax+b
	Timer t_array("ordinary arrays",cout); // starts a timer
 	for (int it=3D1; it <=3D nLoops; ++it)
	{
		for (int i =3D 0; i < NumPart; ++i)
			for (int d =3D 0; d < nsd; ++d)
				pc.y_[i][d] =3D pc.a_[i]*pc.x_[i][d] +
pc.b_[i];
	}
	t_array.stop();
	t_array.print();

 // using pooma attributes: y =3D ax+b
	Timer t_PoomaAttributes("pooma attributes",cout); // starts a
timer
 	for (int it=3D1; it <=3D nLoops; ++it)
	{
		pc.y =3D pc.a*pc.x + pc.b;
	}
	t_PoomaAttributes.stop();
	t_PoomaAttributes.print();

 // Shut down POOMA and exit
	Pooma::finalize();
	return 0;
}




From oldham at codesourcery.com  Tue Dec  4 20:43:13 2001
From: oldham at codesourcery.com (Jeffrey Oldham)
Date: Tue, 4 Dec 2001 12:43:13 -0800
Subject: Explanation of blockAndEvaluate()
Message-ID: <20011204124313.A6159@codesourcery.com>

Mark requested that Stephen Smith's explanation be posted to the
pooma-dev mailing list so it is archived for posterity.

Jeffrey's complaint:
> When I run the attached Pooma program (from examples/Manual/Doof2d/)
> for one-processor, it works fine, returning 55.0221 for 4 averagings
> and an array size of 20.  When I run it with Pooma configured with
> --messaging and use the MM Shared Memory Library, it returns 0.  Just
> before the blockAndEvaluate() call, the "b" array has the proper value
> but afterwards it has changed to zero.  Why?  Why is it ever dangerous
> to call blockAndEvaluate()?  How do I explain when to call
> blockAndEvaluate()?

The program is attached.

Stephen Smith's (stephens at proximation.com) reply:
> This code is missing a blockAndEvaluate, it should look
> like:
>  
> a = b = 0;
> Pooma::blockAndEvaluate();
> b(n/2,n/2) = 1000.0;
> 
> Currently the default is that all code is dangerous, which may
> not be a good thing.  To ensure correctness you either need
> to run with --poomaBlockingExpressions or add blockAndEvaluate()
> all the necessary places.
> 
> Here's the basic issue:
> 
> 1: a = b;
> 2: c = a;
> 3: e = c;
> 4: c(5) = 7;
> 5: d = c + e;
> 6: cout << d(5) << d(3) << endl;
> 
> For this code to work correctly, the data-parallel expressions
> writing to c must be done before statement 4 is run and the
> data-parallel expression writing to d must be done before the
> line that prints values from d.  Using blockingExpressions()
> ensures correctness by inserting blockAndEvaluate() after EVERY
> data-parallel statement:
> 
> 1: a = b;
>    blockAndEvaluate();
> 2: c = a;
>    blockAndEvaluate();
> 3: e = c;
>    blockAndEvaluate();
> 4: c(5) = 7;
> 5: d = c + e;
>    blockAndEvaluate();
> 6: cout << d(5) << d(3) << endl;
> 
> This may not be very efficient when the arrays are decomposed
> into patches, because all the patches in statement 1 must execute
> before any from statement 2.  It would be a lot more cache efficient
> to perform (a = b; c = a; e = c;) on one patch, then move to the next
> patch.
> 
> In the past, my recommendation to users was to add blockAndEvaluate
> immediately before any serial code:
> 
> 1: a = b;
> 2: c = a;
> 3: e = c;
>    blockAndEvaluate();
> 4: c(5) = 7;
> 5: d = c + e;
>    blockAndEvaluate();
> 6: cout << d(5) << d(3) << endl;
> 
> This approach is guaranteed to ensure correctness.  There was no
> way for use to implement this automatically.  We know inside POOMA
> every time a data-parallel expression occurs, but we don't know what
> the next statement is going to be.  There's no simple way to check for
> serial access without slowing the code down incredibly.  All the inner
> loops which get run by SMARTS also access elements through operator(),
> so we would have to put an if test for every element access that would
> say "Are we running inside the evaluator, or back in the users code?"
> 
> So the use of blockAndEvaluate is an optimization.  Perhaps it would be
> better to make --blockingExpressions the default and if users want more
> efficient code they can add the necessary blockAndEvaluates and run
> --withoutBlockingExpressions.  Note that if they really understand
> the parallelism issues, they could get trickier:
>  
> 1: a = b;
> 2: c = a;
>    blockAndEvaluate();
> 3: e = c;
> 4: c(5) = 7;
> 5: d = c + e;
>    blockAndEvaluate();
> 6: cout << d(5) << d(3) << endl;
> 
> is also correct because we've guaranteed that c has been computed.  Note
> that blockAndEvaluate() causes EVERY expression to finally be computed.
> We had at one point thought about a more specific syntax:
> 
> blockOnEvaluation(c);
> c(5) = 7;
> 
> This syntax would ensure that all the expressions relating to a given
> array are finished.  (That would allow the main branch of the code to
> continue while some computations are still going.)
> 
> This idea is a ways off from even being prototyped, though.

Thanks,
Jeffrey D. Oldham
oldham at codesourcery.com
-------------- next part --------------
#include <iostream>		// has std::cout, ...
#include <stdlib.h>		// has EXIT_SUCCESS
#include "Pooma/Arrays.h"	// has Pooma's Array

// Doof2d: Pooma Arrays, element-wise implementation

int main(int argc, char *argv[])
{
  // Prepare the Pooma library for execution.
  Pooma::initialize(argc,argv);
  
  // Ask the user for the number of averagings.
  long nuAveragings, nuIterations;
  std::cout << "Please enter the number of averagings: ";
  std::cin >> nuAveragings;
  nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.

  // Ask the user for the number n of elements along one dimension of
  // the grid.
  long n;
  std::cout << "Please enter the array size: ";
  std::cin >> n;

  // Specify the arrays' domains [0,n) x [0,n).
  Interval<1> N(0, n-1);
  Interval<2> vertDomain(N, N);

  // Create the arrays.
  // The template parameters indicate 2 dimensions, a 'double' element
  // type, and ordinary 'Brick' storage.
  Array<2, double, Brick> a(vertDomain);
  Array<2, double, Brick> b(vertDomain);

  // Set up the initial conditions.
  // All grid values should be zero except for the central value.
  a = b = 0.0;
  b(n/2,n/2) = 1000.0;

  // In the average, weight element with this value.
  const double weight = 1.0/9.0;

  // Perform the simulation.
  for (int k = 0; k < nuIterations; ++k) {
    // Read from b.  Write to a.
    for (int j = 1; j < n-1; j++)
      for (int i = 1; i < n-1; i++)
        a(i,j) = weight *
          (b(i+1,j+1) + b(i+1,j  ) + b(i+1,j-1) +
           b(i  ,j+1) + b(i  ,j  ) + b(i  ,j-1) +
           b(i-1,j+1) + b(i-1,j  ) + b(i-1,j-1));

    // Read from a.  Write to b.
    for (int j = 1; j < n-1; j++)
      for (int i = 1; i < n-1; i++)
        b(i,j) = weight *
          (a(i+1,j+1) + a(i+1,j  ) + a(i+1,j-1) +
           a(i  ,j+1) + a(i  ,j  ) + a(i  ,j-1) +
           a(i-1,j+1) + a(i-1,j  ) + a(i-1,j-1));
  }

  // Print out the final central value.
  std::cout << "before: " << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl; // TMP
  Pooma::blockAndEvaluate();	// Ensure all computation has finished.
  std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;

  // The arrays are automatically deallocated.

  // Tell the Pooma library execution has finished.
  Pooma::finalize();
  return EXIT_SUCCESS;
}

From stephens at proximation.com  Tue Dec  4 20:01:04 2001
From: stephens at proximation.com (Stephen Smith)
Date: Tue, 4 Dec 2001 13:01:04 -0700 
Subject: [pooma-dev] Explanation of blockAndEvaluate()
Message-ID: <F871BB04B0EA224A821A5274C31DF3150217FD@proxim-sf001.santafe.proximation.com>

Actually, Jim pointed out a problem with one of the examples.
I was trying to come up with an example where code that doesn't
need to be evaluated can occur between blockAndEvaluate and some
serial code.  You want everything that reads as
well as writes to a given array to be done before you change
it, and I missed one of the dependencies.

-----Original Message-----
From: Jeffrey Oldham [mailto:oldham at codesourcery.com]
Sent: Tuesday, December 04, 2001 1:43 PM
To: pooma-dev at pooma.codesourcery.com
Subject: [pooma-dev] Explanation of blockAndEvaluate()
 
> 1: a = b;
> 2: c = a;
>    blockAndEvaluate();
> 3: e = c;
> 4: c(5) = 7;
> 5: d = c + e;
>    blockAndEvaluate();
> 6: cout << d(5) << d(3) << endl;

this isn't quite correct because statement 3 reads from c and
statement 4 could occur before statement 3.  The following example
is correct:

1: a = b;
2: c = a;
   blockAndEvaluate();
3: e = b;
4: c(5) = 7;
5: d = c + e;
   blockAndEvaluate();
6: cout << d(5) << d(3) << endl;

It's tricky to figure out, which is why it's best to just put
blockAndEvaluate() immediately before all blocks of serial access,
if you want code that is pretty close to optimal and guaranteed to
be correct.

    Stephen
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://sourcerytools.com/pipermail/pooma-dev/attachments/20011204/37496bc1/attachment.html>

From oldham at codesourcery.com  Tue Dec 11 20:25:01 2001
From: oldham at codesourcery.com (Jeffrey Oldham)
Date: Tue, 11 Dec 2001 12:25:01 -0800
Subject: Patch: Revise Manual Example Codes
Message-ID: <20011211122501.A28891@codesourcery.com>

The example (tutorial) programs changed to become more consistent,
easier to use, and, most importantly, more correct.  These are stored
in examples/Manual/Doof2d/.

2001-Dec-11  Jeffrey D. Oldham  <oldham at codesourcery.com>

	* Doof2d-Array-distributed.cpp: Remove <iostream>, which is not
	used.
	(DoofNinePt): Fix typo in comment.
	(main): Revise to use command-line arguments and Informs, not
	standard IO.  Modify to ensure domain size is a multiple of the
	number of processors.  Add blockAndEvaluate().
	* Doof2d-Array-element.cpp (main): Replace data-parallel
	initialization with loops.  Fix typo in comments.  Remove
	unnecessary blockAndEvaluate().
	* Doof2d-Array-parallel.cpp (main): Add
	blockAndEvaluate().  Fix typo in comment.
	* Doof2d-Array-stencil.cpp (DoofNinePt):
	Fix typo in comment.
	(main): Add blockAndEvaluate().
	* Doof2d-C-element.cpp (main): Fix typo in comment.
	* Doof2d-Field-distributed.cpp: Remove <iostream>, which is not
	used.
	(main): Revise to use command-line arguments and Informs, not
	standard IO.  Modify to ensure domain size is a multiple of the
	number of processors.  Add blockAndEvaluate().  Fix typo in
	comment.
	* Doof2d-Field-parallel.cpp (main): Revise input to use cin, not
	hard-coded constants.  Add blockAndEvaluate().  Fix typo in
	comment.

Applied to	mainline

Thanks,
Jeffrey D. Oldham
oldham at codesourcery.com
-------------- next part --------------
Index: Doof2d-Array-distributed.cpp
===================================================================
RCS file: /home/pooma/Repository/r2/examples/Manual/Doof2d/Doof2d-Array-distributed.cpp,v
retrieving revision 1.1
diff -c -p -r1.1 Doof2d-Array-distributed.cpp
*** Doof2d-Array-distributed.cpp	2001/12/04 00:07:00	1.1
--- Doof2d-Array-distributed.cpp	2001/12/11 19:01:38
***************
*** 1,4 ****
- #include <iostream>		// has std::cout, ...
  #include <stdlib.h>		// has EXIT_SUCCESS
  #include "Pooma/Arrays.h"	// has Pooma's Array
  
--- 1,3 ----
*************** public:
*** 30,36 ****
  
  private:
  
!   // In the average, weight element with this value.
    const double weight;
  };
  
--- 29,35 ----
  
  private:
  
!   // In the average, weight elements with this value.
    const double weight;
  };
  
*************** int main(int argc, char *argv[])
*** 38,60 ****
  {
    // Prepare the Pooma library for execution.
    Pooma::initialize(argc,argv);
!   
!   // Ask the user for the number of processors.
    long nuProcessors;
!   std::cout << "Please enter the number of processors: ";
!   std::cin >> nuProcessors;
  
!   // Ask the user for the number of averagings.
    long nuAveragings, nuIterations;
!   std::cout << "Please enter the number of averagings: ";
!   std::cin >> nuAveragings;
    nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
  
    // Ask the user for the number n of elements along one dimension of
    // the grid.
    long n;
!   std::cout << "Please enter the array size: ";
!   std::cin >> n;
  
    // Specify the arrays' domains [0,n) x [0,n).
    Interval<1> N(0, n-1);
--- 37,73 ----
  {
    // Prepare the Pooma library for execution.
    Pooma::initialize(argc,argv);
! 
!   // Since multiple copies of this program may simultaneously run, we
!   // canot use standard input and output.  Instead we use command-line
!   // arguments, which are replicated, for input, and we use an Inform
!   // stream for output.
!   Inform output;
! 
!   // Read the program input from the command-line arguments.
!   if (argc != 4) {
!     // Incorrect number of command-line arguments.
!     output << argv[0] << ": number-of-processors number-of-averagings number-of-values" << std::endl;
!     return EXIT_FAILURE;
!   }
!   char *tail;
! 
!   // Determine the number of processors.
    long nuProcessors;
!   nuProcessors = strtol(argv[1], &tail, 0);
  
!   // Determine the number of averagings.
    long nuAveragings, nuIterations;
!   nuAveragings = strtol(argv[2], &tail, 0);
    nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
  
    // Ask the user for the number n of elements along one dimension of
    // the grid.
    long n;
!   n = strtol(argv[3], &tail, 0);
!   // The dimension must be a multiple of the number of processors
!   // since we are using a UniformGridLayout.
!   n = ((n+nuProcessors-1) / nuProcessors) * nuProcessors;
  
    // Specify the arrays' domains [0,n) x [0,n).
    Interval<1> N(0, n-1);
*************** int main(int argc, char *argv[])
*** 85,90 ****
--- 98,105 ----
    // Set up the initial conditions.
    // All grid values should be zero except for the central value.
    a = b = 0.0;
+   // Ensure all data-parallel computation finishes before accessing a value.
+   Pooma::blockAndEvaluate();
    b(n/2,n/2) = 1000.0;
  
    // Create the stencil performing the computation.
*************** int main(int argc, char *argv[])
*** 101,107 ****
  
    // Print out the final central value.
    Pooma::blockAndEvaluate();	// Ensure all computation has finished.
!   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
  
    // The arrays are automatically deallocated.
  
--- 116,122 ----
  
    // Print out the final central value.
    Pooma::blockAndEvaluate();	// Ensure all computation has finished.
!   output << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
  
    // The arrays are automatically deallocated.
  
Index: Doof2d-Array-element.cpp
===================================================================
RCS file: /home/pooma/Repository/r2/examples/Manual/Doof2d/Doof2d-Array-element.cpp,v
retrieving revision 1.1
diff -c -p -r1.1 Doof2d-Array-element.cpp
*** Doof2d-Array-element.cpp	2001/12/04 00:07:00	1.1
--- Doof2d-Array-element.cpp	2001/12/11 19:01:38
*************** int main(int argc, char *argv[])
*** 33,42 ****
  
    // Set up the initial conditions.
    // All grid values should be zero except for the central value.
!   a = b = 0.0;
    b(n/2,n/2) = 1000.0;
  
!   // In the average, weight element with this value.
    const double weight = 1.0/9.0;
  
    // Perform the simulation.
--- 33,44 ----
  
    // Set up the initial conditions.
    // All grid values should be zero except for the central value.
!   for (int j = 1; j < n-1; j++)
!     for (int i = 1; i < n-1; i++)
!       a(i,j) = b(i,j) = 0.0;
    b(n/2,n/2) = 1000.0;
  
!   // In the average, weight elements with this value.
    const double weight = 1.0/9.0;
  
    // Perform the simulation.
*************** int main(int argc, char *argv[])
*** 59,65 ****
    }
  
    // Print out the final central value.
-   Pooma::blockAndEvaluate();	// Ensure all computation has finished.
    std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
  
    // The arrays are automatically deallocated.
--- 61,66 ----
Index: Doof2d-Array-parallel.cpp
===================================================================
RCS file: /home/pooma/Repository/r2/examples/Manual/Doof2d/Doof2d-Array-parallel.cpp,v
retrieving revision 1.1
diff -c -p -r1.1 Doof2d-Array-parallel.cpp
*** Doof2d-Array-parallel.cpp	2001/12/04 00:07:00	1.1
--- Doof2d-Array-parallel.cpp	2001/12/11 19:01:38
*************** int main(int argc, char *argv[])
*** 38,46 ****
    // Set up the initial conditions.
    // All grid values should be zero except for the central value.
    a = b = 0.0;
    b(n/2,n/2) = 1000.0;
  
!   // In the average, weight element with this value.
    const double weight = 1.0/9.0;
  
    // Perform the simulation.
--- 38,48 ----
    // Set up the initial conditions.
    // All grid values should be zero except for the central value.
    a = b = 0.0;
+   // Ensure all data-parallel computation finishes before accessing a value.
+   Pooma::blockAndEvaluate();
    b(n/2,n/2) = 1000.0;
  
!   // In the average, weight elements with this value.
    const double weight = 1.0/9.0;
  
    // Perform the simulation.
Index: Doof2d-Array-stencil.cpp
===================================================================
RCS file: /home/pooma/Repository/r2/examples/Manual/Doof2d/Doof2d-Array-stencil.cpp,v
retrieving revision 1.1
diff -c -p -r1.1 Doof2d-Array-stencil.cpp
*** Doof2d-Array-stencil.cpp	2001/12/04 00:07:00	1.1
--- Doof2d-Array-stencil.cpp	2001/12/11 19:01:38
*************** public:
*** 30,36 ****
  
  private:
  
!   // In the average, weight element with this value.
    const double weight;
  };
  
--- 30,36 ----
  
  private:
  
!   // In the average, weight elements with this value.
    const double weight;
  };
  
*************** int main(int argc, char *argv[])
*** 68,73 ****
--- 68,75 ----
    // Set up the initial conditions.
    // All grid values should be zero except for the central value.
    a = b = 0.0;
+   // Ensure all data-parallel computation finishes before accessing a value.
+   Pooma::blockAndEvaluate();
    b(n/2,n/2) = 1000.0;
  
    // Create the stencil performing the computation.
Index: Doof2d-C-element.cpp
===================================================================
RCS file: /home/pooma/Repository/r2/examples/Manual/Doof2d/Doof2d-C-element.cpp,v
retrieving revision 1.1
diff -c -p -r1.1 Doof2d-C-element.cpp
*** Doof2d-C-element.cpp	2001/12/04 00:07:00	1.1
--- Doof2d-C-element.cpp	2001/12/11 19:01:38
*************** int main()
*** 37,43 ****
        a[i][j] = b[i][j] = 0.0;
    b[n/2][n/2] = 1000.0;
  
!   // In the average, weight element with this value.
    const double weight = 1.0/9.0;
  
    // Perform the simulation.
--- 37,43 ----
        a[i][j] = b[i][j] = 0.0;
    b[n/2][n/2] = 1000.0;
  
!   // In the average, weight elements with this value.
    const double weight = 1.0/9.0;
  
    // Perform the simulation.
Index: Doof2d-Field-distributed.cpp
===================================================================
RCS file: /home/pooma/Repository/r2/examples/Manual/Doof2d/Doof2d-Field-distributed.cpp,v
retrieving revision 1.1
diff -c -p -r1.1 Doof2d-Field-distributed.cpp
*** Doof2d-Field-distributed.cpp	2001/12/04 00:07:00	1.1
--- Doof2d-Field-distributed.cpp	2001/12/11 19:01:38
***************
*** 1,4 ****
- #include <iostream>		// has std::cout, ...
  #include <stdlib.h>		// has EXIT_SUCCESS
  #include "Pooma/Fields.h"	// has Pooma's Field
  
--- 1,3 ----
*************** int main(int argc, char *argv[])
*** 9,28 ****
    // Prepare the Pooma library for execution.
    Pooma::initialize(argc,argv);
    
!   // nuIterations is the number of simulation iterations.
!   const int nuIterations = 10/2;
! 
!   // In the average, weight element with this value.
!   const double weight = 1.0/9.0;
  
!   // nuProcessors is the number of processors along one dimension.
!   const int nuProcessors = 2;
  
    // Ask the user for the number n of elements along one dimension of
    // the grid.
    long n;
!   std::cout << "Please enter the array size: ";
!   std::cin >> n;
  
    // Specify the fields' domains [0,n) x [0,n).
    Interval<1> N(0, n-1);
--- 8,43 ----
    // Prepare the Pooma library for execution.
    Pooma::initialize(argc,argv);
    
!   // Since multiple copies of this program may simultaneously run, we
!   // canot use standard input and output.  Instead we use command-line
!   // arguments, which are replicated, for input, and we use an Inform
!   // stream for output.
!   Inform output;
! 
!   // Read the program input from the command-line arguments.
!   if (argc != 4) {
!     // Incorrect number of command-line arguments.
!     output << argv[0] << ": number-of-processors number-of-averagings number-of-values" << std::endl;
!     return EXIT_FAILURE;
!   }
!   char *tail;
  
!   // Determine the number of processors.
!   long nuProcessors;
!   nuProcessors = strtol(argv[1], &tail, 0);
! 
!   // Determine the number of averagings.
!   long nuAveragings, nuIterations;
!   nuAveragings = strtol(argv[2], &tail, 0);
!   nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
  
    // Ask the user for the number n of elements along one dimension of
    // the grid.
    long n;
!   n = strtol(argv[3], &tail, 0);
!   // The dimension must be a multiple of the number of processors
!   // since we are using a UniformGridLayout.
!   n = ((n+nuProcessors-1) / nuProcessors) * nuProcessors;
  
    // Specify the fields' domains [0,n) x [0,n).
    Interval<1> N(0, n-1);
*************** int main(int argc, char *argv[])
*** 59,66 ****
--- 74,86 ----
    // Set up the initial conditions.
    // All grid values should be zero except for the central value.
    a = b = 0.0;
+   // Ensure all data-parallel computation finishes before accessing a value.
+   Pooma::blockAndEvaluate();
    b(n/2,n/2) = 1000.0;
  
+   // In the average, weight elements with this value.
+   const double weight = 1.0/9.0;
+ 
    // Perform the simulation.
    for (int k = 0; k < nuIterations; ++k) {
      // Read from b.  Write to a.
*************** int main(int argc, char *argv[])
*** 77,83 ****
    }
  
    // Print out the final central value.
!   std::cout << b(n/2,n/2) << std::endl;
  
    // The fields are automatically deallocated.
  
--- 97,104 ----
    }
  
    // Print out the final central value.
!   Pooma::blockAndEvaluate();	// Ensure all computation has finished.
!   output << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
  
    // The fields are automatically deallocated.
  
Index: Doof2d-Field-parallel.cpp
===================================================================
RCS file: /home/pooma/Repository/r2/examples/Manual/Doof2d/Doof2d-Field-parallel.cpp,v
retrieving revision 1.1
diff -c -p -r1.1 Doof2d-Field-parallel.cpp
*** Doof2d-Field-parallel.cpp	2001/12/04 00:07:00	1.1
--- Doof2d-Field-parallel.cpp	2001/12/11 19:01:38
*************** int main(int argc, char *argv[])
*** 9,24 ****
    // Prepare the Pooma library for execution.
    Pooma::initialize(argc,argv);
    
!   // nuIterations is the number of simulation iterations.
!   const int nuIterations = 10/2;
  
-   // In the average, weight element with this value.
-   const double weight = 1.0/9.0;
- 
    // Ask the user for the number n of elements along one dimension of
    // the grid.
    long n;
!   std::cout << "Please enter the array size: ";
    std::cin >> n;
  
    // Specify the fields' domains [0,n) x [0,n).
--- 9,24 ----
    // Prepare the Pooma library for execution.
    Pooma::initialize(argc,argv);
    
!   // Ask the user for the number of averagings.
!   long nuAveragings, nuIterations;
!   std::cout << "Please enter the number of averagings: ";
!   std::cin >> nuAveragings;
!   nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
  
    // Ask the user for the number n of elements along one dimension of
    // the grid.
    long n;
!   std::cout << "Please enter the field size: ";
    std::cin >> n;
  
    // Specify the fields' domains [0,n) x [0,n).
*************** int main(int argc, char *argv[])
*** 44,51 ****
--- 44,56 ----
    // Set up the initial conditions.
    // All grid values should be zero except for the central value.
    a = b = 0.0;
+   // Ensure all data-parallel computation finishes before accessing a value.
+   Pooma::blockAndEvaluate();
    b(n/2,n/2) = 1000.0;
  
+   // In the average, weight elements with this value.
+   const double weight = 1.0/9.0;
+ 
    // Perform the simulation.
    for (int k = 0; k < nuIterations; ++k) {
      // Read from b.  Write to a.
*************** int main(int argc, char *argv[])
*** 62,68 ****
    }
  
    // Print out the final central value.
!   std::cout << b(n/2,n/2) << std::endl;
  
    // The fields are automatically deallocated.
  
--- 67,74 ----
    }
  
    // Print out the final central value.
!   Pooma::blockAndEvaluate();	// Ensure all computation has finished.
!   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
  
    // The fields are automatically deallocated.

From oldham at codesourcery.com  Tue Dec 11 21:40:31 2001
From: oldham at codesourcery.com (Jeffrey Oldham)
Date: Tue, 11 Dec 2001 13:40:31 -0800
Subject: Patch: Manual: Tutorial Chapter and Part of Concepts Chapter
Message-ID: <20011211134031.A29015@codesourcery.com>

This patch contains continuing work toward a Pooma manual.  Most of
these changes fail into these two categories:

1) continuing work on a tutorial chapter using Doof2d implementations
   to introduce Pooma concepts
2) moving material among files to yield smaller files.

2001-Dec-11  Jeffrey D. Oldham  <oldham at codesourcery.com>

	* glossary.xml: New file containing the glossary entries.  It was
	previously part of outline.xml.
	* makefile: Revise to create manual.ps, not outline.ps.
	* manual.xml: New file containing most of the manual.  It replaces
	outline.xml.  Too many changes were made to be listed here.  Most
	of the changes have occurred in the tutorial chapter and the
	concept chapter.
	* outline.xml: Removed in favor of manual.xml and the files the
	latter includes.
	* tutorial.xml: New file containing the tutorial chapter.  This
	was created from outline.xml.
	* figures/concepts.mp: New MetaPost file to create illustrations
	for Array and Field's purposes and of the conceptual dependences
	when declaring containers.
	* figures/distributed.mp: Revise to make it smaller, move macros
	out of the source code, and fix formatting mistake.
	* figures/doof2d.mp: Revise to use macros.
	* figures/macros.ltx: New file containing LaTeX macros common to
	MetaPost files.
	* programs/Doof2d-Array-distributed-annotated.patch: Revised to
	reflect changes to programs/Doof2d-Array-distributed.cpp.
	* programs/Doof2d-Array-distributed-annotated.patch: Similar.
	* programs/Doof2d-Array-parallel-annotated.patch: Similar.
	* programs/Doof2d-Array-stencil-annotated.patch: Similar.
	* programs/Doof2d-C-element-annotated.patch: Similar.
	* programs/Doof2d-Field-distributed-annotated.patch: New file
	with an annotated version of Doof2d-Field-distributed.cpp.
	* programs/Doof2d-Field-parallel-annotated.patch: Similar.

Applied to	mainline

Thanks,
Jeffrey D. Oldham
oldham at codesourcery.com
-------------- next part --------------
Index: glossary.xml
===================================================================
RCS file: glossary.xml
diff -N glossary.xml
*** /dev/null	Fri Mar 23 21:37:44 2001
--- glossary.xml	Tue Dec 11 13:31:06 2001
***************
*** 0 ****
--- 1,514 ----
+  <!-- Glossary -->
+ 
+ <glossary id="glossary">
+  <title>Glossary</title>
+ 
+  <!-- Write the definitions so reading
+   1) the term
+   2) an implicit "is (a/an)"
+   3) the definition is a complete paragraph.
+   Thus, the beginning of the definition should not be capitalized and
+   should finish a sentence.  The rest of the definition should
+   consist of complete sentences.  -->
+ 
+  <para>ADD: Make sure all entries are indexed and perhaps point back
+  to their first use.  WARNING: This is constructed by hand so it is
+  likely to be full of inconsistencies and errors.</para>
+ 
+  <glossdiv id="glossary-a">
+   <title>A</title>
+   <glossentry id="glossary-array">
+    <glossterm>&array;</glossterm>
+    <glossdef>
+     <para>a &pooma; container generalizing &c; arrays and mapping
+     indices to values.  Constant-time access to values is supported,
+     ignoring the time to compute the values if applicable.  &array;s
+     are <link linkend="glossary-first_class">first-class
+     object</link>s.  <link
+ 			    linkend="glossary-dynamicarray">&dynamicarray;</link>s and <link
+ 											     linkend="glossary-field">&field;</link>s generalize &array;.</para>
+     <glossseealso otherterm="glossary-dynamicarray">&dynamicarray;</glossseealso>
+     <glossseealso otherterm="glossary-field">&field;</glossseealso>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+  <glossdiv id="glossary-b">
+   <title>B</title>
+   <glossentry id="glossary-brick">
+    <glossterm>brick engine</glossterm>
+    <glossdef>
+     <para>an engine explicitly storing each of its values.  Its space
+     requirements are at least the size of the engine's domain.</para>
+     <glossseealso otherterm="glossary-engine">engine</glossseealso>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+  <glossdiv id="glossary-c">
+   <title>C</title>
+ 
+   <glossentry id="glossary-cell">
+    <glossterm>cell</glossterm>
+    <glossdef>
+     <para>a <link linkend="glossary-domain">domain</link> element of
+     a <link linkend="glossary-field">&field;</link>.  Both <link
+ 								  linkend="glossary-array">&array;</link> and &field; domain
+     elements are denoted by indices, but a cell exists in space.  For
+     example, it might be a rectangle or rectangular
+     parallelepiped.</para>
+     <glossseealso otherterm="glossary-field">&field;</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-cell_size">
+    <glossterm>cell size</glossterm>
+    <glossdef>
+     <para>specifies a &field; cell's dimensions e.g., its
+     width, height, and depth, in&nbsp;&space;.  This is frequently
+     used to specify a mesh.</para>
+     <glossseealso otherterm="glossary-mesh">mesh</glossseealso>
+     <glossseealso otherterm="glossary-corner_position">corner position</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-computing_environment">
+    <glossterm>computing environment</glossterm>
+    <glossdef>
+     <para>computer.  More precisely, a computer with its arrangement
+     of processors and associated memory, possibly shared among
+     processors.</para>
+     <glossseealso otherterm="glossary-sequential">sequential computing environment</glossseealso>
+     <glossseealso otherterm="glossary-distributed">distributed computing environment</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-container">
+    <glossterm>container</glossterm>
+    <glossdef>
+     <para>an object that stores other objects, controlling their
+     allocation, deallocation, and access.  Similar to &cc; containers,
+     the most important &pooma; containers are <link
+     linkend="glossary-array">&array;</link>s and <link
+     linkend="glossary-field">&field;</link>s.</para>
+     <glossseealso otherterm="glossary-array">&array;</glossseealso>
+     <glossseealso otherterm="glossary-dynamicarray">&dynamicarray;</glossseealso>
+     <glossseealso otherterm="glossary-field">&field;</glossseealso>
+     <glossseealso otherterm="glossary-tensor">&tensor;</glossseealso>
+     <glossseealso otherterm="glossary-matrix">&matrix;</glossseealso>
+     <glossseealso otherterm="glossary-vector">&vector;</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-context">
+    <glossterm>context</glossterm>
+    <glossdef>
+     <para>a collection of shared memory and processors that can execute
+      a program of a portion of a program.  It can have one or more
+      processors, but all these processors must access the same shared
+      memory.  Usually the computer and its operating system, not the
+      programmer, determine the available contexts.</para>
+     <glossseealso otherterm="glossary-distributed">distributed computing environment</glossseealso>
+     <glossseealso otherterm="glossary-layout">layout</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-context_mapper">
+    <glossterm>context mapper</glossterm>
+    <glossdef>
+     <para>indicates how a container's patches are mapped to
+     processors and shared memory.  Two common choices are
+     distribution among the various processors and replication.</para>
+     <glossseealso otherterm="glossary-patch">patch</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-corner_position">
+    <glossterm>corner position</glossterm>
+    <glossdef>
+     <para>specifies the &space; point corresponding to a &field;
+     domain's lower, left corner.</para>
+     <glossseealso otherterm="glossary-mesh">mesh</glossseealso>
+     <glossseealso otherterm="glossary-cell_size">cell size</glossseealso>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+  <glossdiv id="glossary-d">
+   <title>D</title>
+ 
+   <glossentry id="glossary-distributed">
+    <glossterm>distributed computing environment</glossterm>
+    <glossdef>
+     <para>computing environment with one or more processors each
+     having associated memory, possibly shared.  In some contexts, it
+     refers to strictly multi-processor computation.</para>
+     <glossseealso otherterm="glossary-computing_environment">computing environment</glossseealso>
+     <glossseealso otherterm="glossary-sequential">sequential computing environment</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-domain">
+    <glossterm>domain</glossterm>
+    <glossdef>
+     <para>a set of points on which a container can define values.
+     For example, a set of discrete integral &n;-tuples in
+     &n;-dimensional space frequently serve as container domains.</para>
+     <glossseealso otherterm="glossary-container">container</glossseealso>
+     <glossseealso otherterm="glossary-interval">interval</glossseealso>
+     <glossseealso otherterm="glossary-stride">stride</glossseealso>
+     <glossseealso otherterm="glossary-range">range</glossseealso>
+     <glossseealso otherterm="glossary-region">region</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-dynamicarray">
+    <glossterm>&dynamicarray;</glossterm>
+    <glossdef>
+     <para>a &pooma; container generalizing one-dimensional <link
+ 								  linkend="glossary-array">&array;</link>s by supporting domain
+     resizing at run-time.  It maps indices to values in constant-time
+     access, ignoring the time to compute the values if applicable.
+     &dynamicarray;s are <link
+ 			       linkend="glossary-first_class">first-class object</link>s.</para>
+     <glossseealso otherterm="glossary-array">&array;</glossseealso>
+     <glossseealso otherterm="glossary-field">&field;</glossseealso>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+   <glossdiv id="glossary-e">
+   <title>E</title>
+ 
+   <glossentry id="glossary-engine">
+    <glossterm>engine</glossterm>
+    <glossdef>
+     <para>stores and, if necessary, computes a <link
+ 						      linkend="glossary-container">container</link>'s values.  These
+     can be specialized, e.g., to minimize storage when a domain has
+     few distinct values.  Separating a container and its storage also
+     permits <link linkend="glossary-view">views</link> of a container.</para>
+     <glossseealso otherterm="glossary-container">&engine;</glossseealso>
+     <glossseealso otherterm="glossary-view">view of a container</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-external_guard_layer">
+    <glossterm>external guard layer</glossterm>
+    <glossdef>
+     <para><link linkend="glossary-guard_layer">guard layer</link>
+     surrounding a container's domain used to ease computation along
+     the domain's edges by permitting the same computations as for
+     more internal computations.  It is an optimization, not required
+     for program correctness.</para>
+     <glossseealso otherterm="glossary-guard_layer">guard layer</glossseealso>
+     <glossseealso otherterm="glossary-internal_guard_layer">internal guard layer</glossseealso>
+     <glossseealso otherterm="glossary-patch">patch</glossseealso>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+  <glossdiv id="glossary-f">
+   <title>F</title>
+ 
+   <glossentry id="glossary-field">
+    <glossterm>&field;</glossterm>
+    <glossdef>
+     <para>a &pooma; container representing an &array; with spatial
+     extent.  It also supports multiple values and multiple materials
+     indexed by the same value.  It maps indices to values in constant
+     time, ignoring the time to compute the values if applicable.  It
+     also supports geometric computations such as the distance between
+     two <link linkend="glossary-cell">cell</link>s and normals to a
+     cell.  &field;s are <link
+ 			       linkend="glossary-first_class">first-class object</link>s.</para>
+     <glossseealso otherterm="glossary-array">&array;</glossseealso>
+     <glossseealso otherterm="glossary-dynamicarray">&dynamicarray;</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-first_class">
+    <glossterm>first-class type</glossterm>
+    <glossdef>
+     <para>a type of object with all the capabilities of the built-in
+     type with the most capabilities.  For example, <type>char</type>
+     and <type>int</type> are first-class types in &cc; because they
+     may be declared anywhere, stored in automatic variables, accessed
+     anywhere, copied, and passed by both value and reference.
+     &pooma; <link linkend="glossary-array">&array;</link> and <link
+ 								     linkend="glossary-field">&field;</link> are first-class
+     types.</para>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+  <glossdiv id="glossary-g">
+   <title>G</title>
+ 
+   <glossentry id="glossary-guard_layer">
+    <glossterm>guard layer</glossterm>
+    <glossdef>
+     <para>domain surrounding each patch of a container's domain.  It
+     contains read-only values.  <link
+ 				       linkend="glossary-external_guard_layer">External guard
+     layer</link>s ease programming, while <link
+ 						 linkend="glossary-internal_guard_layer">internal guard
+     layer</link>s permit each patch's computation to be occur without
+     copying values from adjacent patches.  They are optimizations,
+     not required for program correctness.</para>
+     <glossseealso otherterm="glossary-external_guard_layer">external guard layer</glossseealso>
+     <glossseealso otherterm="glossary-internal_guard_layer">internal guard layer</glossseealso>
+     <glossseealso otherterm="glossary-partition">partition</glossseealso>
+     <glossseealso otherterm="glossary-patch">patch</glossseealso>
+     <glossseealso otherterm="glossary-domain">domain</glossseealso>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+  <glossdiv id="glossary-i">
+   <title>I</title>
+ 
+   <glossentry id="glossary-index">
+    <glossterm>index</glossterm>
+    <glossdef>
+     <para>a position in a <link
+ 				 linkend="glossary-domain">domain</link> usually denoted by an
+     ordered tuple.  More than one index are called <link
+ 							  linkend="glossary-indices">indices</link>.</para>
+     <glossseealso otherterm="glossary-domain">domain</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-indices">
+    <glossterm>indices</glossterm>
+    <glossdef>
+     <para>More than one <link linkend="glossary-index">index</link>.</para>
+     <glossseealso otherterm="glossary-index">index</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-internal_guard_layer">
+    <glossterm>internal guard layer</glossterm>
+    <glossdef>
+     <para><link linkend="glossary-guard_layer">guard layer</link>
+     containing copies of adjacent patches' values.  These copies can
+     permit an individual patch's computation to occur without asking
+     adjacent patches for values.  This can speed computation but are
+     not required for program correctness.</para>
+     <glossseealso otherterm="glossary-guard_layer">guard layer</glossseealso>
+     <glossseealso otherterm="glossary-external_guard_layer">external guard layer</glossseealso>
+     <glossseealso otherterm="glossary-patch">patch</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-interval">
+    <glossterm>interval</glossterm>
+    <glossdef>
+     <para>a set of integral points between two values.  This domain
+     is frequently represented using mathematical interval notation
+     [a,b] even though it contains only the integral points, e.g., a,
+     a+1, a+2, &hellip;, b.  It is also generalized to the tensor
+     product of one-dimensional intervals.  Many containers' domains
+     consist of these sets of ordered tuples.</para>
+     <glossseealso otherterm="glossary-domain">domain</glossseealso>
+     <glossseealso otherterm="glossary-stride">stride</glossseealso>
+     <glossseealso otherterm="glossary-range">range</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+  </glossdiv>
+ 
+  <glossdiv id="glossary-l">
+   <title>L</title>
+ 
+   <glossentry id="glossary-layout">
+    <glossterm>layout</glossterm>
+    <glossdef>
+     <para>a map from an index to processor(s) and memory used to
+     compute the container's associated value.  For a uniprocessor
+     implementation, a container's layout always consists of its
+     domain, the processor, and its memory.  For a multi-processor
+     implementation, the layout maps portions of the domain to
+     (possibly different) processors and memory.</para>
+     <glossseealso otherterm="glossary-container">container</glossseealso>
+     <glossseealso otherterm="glossary-domain">domain</glossseealso>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+  <glossdiv id="glossary-m">
+   <title>M</title>
+ 
+   <glossentry id="glossary-mesh">
+    <glossterm>mesh</glossterm>
+    <glossdef>
+     <para>a &field;'s map from indices to geometric values such as
+     cell size, edge length, and cell normals.  In other words, it
+     specifies a &field;'s <quote>spatial extent</quote>.</para>
+     <glossseealso otherterm="glossary-field">&field;</glossseealso>
+     <glossseealso otherterm="glossary-layout">layout</glossseealso>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+  <glossdiv id="glossary-p">
+   <title>P</title>
+ 
+   <glossentry id="glossary-partition">
+    <glossterm>partition</glossterm>
+    <glossdef>
+     <para>a specification how to divide a container's domain into
+     patches for distributed computation.  It can be independent of
+     the domain's size.  For example, it divide each domain into
+     halves, yielding a total of eight patches in three dimensions.
+     See <xref
+ 	       linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
+     for an illustration.</para>
+     <glossseealso otherterm="glossary-guard_layer">guard layer</glossseealso>
+     <glossseealso otherterm="glossary-patch">patch</glossseealso>
+     <glossseealso otherterm="glossary-domain">domain</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-patch">
+    <glossterm>patch</glossterm>
+    <glossdef>
+     <para>
+ ERE</para>
+     <glossseealso otherterm="glossary-partition">partition</glossseealso>
+     <glossseealso otherterm="glossary-guard_layer">guard layer</glossseealso>
+     <glossseealso otherterm="glossary-domain">domain</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-point">
+    <glossterm>point</glossterm>
+    <glossdef>
+     <para>a location in multi-dimensional space&nbsp;&space;.
+     In contrast, indices specify positions in container domains.</para>
+     <glossseealso otherterm="glossary-field">&field;</glossseealso>
+     <glossseealso otherterm="glossary-mesh">mesh</glossseealso>
+     <glossseealso otherterm="glossary-index">index</glossseealso>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+  <glossdiv id="glossary-r">
+   <title>R</title>
+ 
+   <glossentry id="glossary-range">
+    <glossterm>range</glossterm>
+    <glossdef>
+     <para>an &n;-dimensional domain formed by the tensor product of
+     &n; <link linkend="glossary-stride">stride</link>s.</para>
+     <glossseealso otherterm="glossary-stride">stride</glossseealso>
+     <glossseealso
+ 		   otherterm="glossary-interval">interval</glossseealso>
+     <glossseealso otherterm="glossary-domain">domain</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-region">
+    <glossterm>region</glossterm>
+    <glossdef>
+     <para>a domain consisting of a subset of &n;-dimensional
+     continuous space.</para>
+     <glossseealso otherterm="glossary-domain">domain</glossseealso>
+     <glossseealso otherterm="glossary-interval">interval</glossseealso>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+  <glossdiv id="glossary-s">
+   <title>S</title>
+ 
+   <glossentry id="glossary-sequential">
+    <glossterm>sequential computing environment</glossterm>
+    <glossdef>
+     <para>a computing environment with one processor and associated
+     memory.  Only one processor executes a program even if the
+     conmputer itself has multiple processors.</para>
+     <glossseealso otherterm="glossary-computing_environment">computing environment</glossseealso>
+     <glossseealso otherterm="glossary-distributed">distributed computing environment</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-stride">
+    <glossterm>stride</glossterm>
+    <glossdef>
+     <para>a subset of regularly-spaced points in an integral
+     interval.  For example, the set of points a, a+2, a+4, &hellip;,
+     b-2, b is specified by [a,b] with stride&nbsp;2.  It is a
+     domain.</para>
+     <glossseealso otherterm="glossary-range">range</glossseealso>
+     <glossseealso otherterm="glossary-interval">interval</glossseealso>
+     <glossseealso otherterm="glossary-domain">domain</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-suite_name">
+    <glossterm>suite name</glossterm>
+    <glossdef>
+     <para>an arbitrary string denoting a particular toolkit
+     configuration.  For example, the string
+     <quote>SUNKCC-debug</quote> might indicate a configuration for
+     the <trademark>Sun</trademark> <productname>Solaris</productname>
+     operating system and the &kcc; &cc; compiler with debugging
+     support.  By default, the suite name it is equal to the
+     configuration's architecture name.</para>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+  <glossdiv id="glossary-t">
+   <title>T</title>
+ 
+   <glossentry id="glossary-tensor">
+    <glossterm>&tensor;</glossterm>
+    <glossdef>
+     <para>a &pooma; container implementing multi-dimensional
+     mathematical tensors as first-class objects.</para>
+     <glossseealso otherterm="glossary-matrix">&matrix;</glossseealso>
+     <glossseealso otherterm="glossary-vector">&vector;</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-matrix">
+    <glossterm>&matrix;</glossterm>
+    <glossdef>
+     <para>a &pooma; container implementing two-dimensional
+     mathematical matrices as first-class objects.</para>
+     <glossseealso otherterm="glossary-tensor">&tensor;</glossseealso>
+     <glossseealso otherterm="glossary-vector">&vector;</glossseealso>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+  <glossdiv id="glossary-v">
+   <title>V</title>
+ 
+   <glossentry id="glossary-vector">
+    <glossterm>&vector;</glossterm>
+    <glossdef>
+     <para>a &pooma; container implementing multi-dimensional
+     mathematical vectors, i.e., an ordered tuple of components, as
+     first-class objects.</para>
+     <glossseealso otherterm="glossary-tensor">&tensor;</glossseealso>
+     <glossseealso otherterm="glossary-matrix">&matrix;</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-view">
+    <glossterm>view of a container</glossterm>
+    <glossdef>
+     <para>a container derived from another.  The former's domain is a
+     subset of the latter's, but, where the domains intersect,
+     accessing a value through the view is the same as accessing it
+     through the original container.  Only &array;s, &dynamicarray;s,
+     and &field;s support views.</para>
+     <glossseealso otherterm="glossary-container">container</glossseealso>
+    </glossdef>
+   </glossentry>
+  </glossdiv>
+ 
+ </glossary>
Index: makefile
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/makefile,v
retrieving revision 1.1
diff -c -p -r1.1 makefile
*** makefile	2001/12/04 00:07:00	1.1
--- makefile	2001/12/11 20:31:06
*************** INDEXOPTIONS=		-t 'Index' -i 'index' -g 
*** 23,29 ****
  
  CXXFLAGS= -g -Wall -pedantic -W -Wstrict-prototypes -Wpointer-arith -Wbad-function-cast -Wcast-align -Wconversion -Wnested-externs -Wundef -Winline -static
  
! all: outline.ps
  
  %.all:	%.ps %.pdf %.html
  	chmod 644 $*.ps $*.pdf
--- 23,29 ----
  
  CXXFLAGS= -g -Wall -pedantic -W -Wstrict-prototypes -Wpointer-arith -Wbad-function-cast -Wcast-align -Wconversion -Wnested-externs -Wundef -Winline -static
  
! all: manual.ps
  
  %.all:	%.ps %.pdf %.html
  	chmod 644 $*.ps $*.pdf
*************** mproof-%.ps:	%.mp
*** 66,69 ****
  	detex $< > $@
  
  clean:
! 	rm -f *.dvi *.aux *.log *.toc *.bak *.blg *.bbl *.glo *.idx *.lof *.lot *.htm *.mpx mpxerr.tex HTML.index outline.tex
--- 66,69 ----
  	detex $< > $@
  
  clean:
! 	rm -f *.dvi *.aux *.log *.toc *.bak *.blg *.bbl *.glo *.idx *.lof *.lot *.htm *.mpx mpxerr.tex HTML.index manual.tex
Index: manual.xml
===================================================================
RCS file: manual.xml
diff -N manual.xml
*** /dev/null	Fri Mar 23 21:37:44 2001
--- manual.xml	Tue Dec 11 13:31:09 2001
***************
*** 0 ****
--- 1,4034 ----
+ <?xml version="1.0"?>
+ 
+ <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN" "file://usr/lib/sgml/docbookx.dtd" [
+ 
+ <!-- Text to Check, Revise, Replace -->
+   <!-- FIXME: Revise before releasing. -->
+   <!-- UPDATE: Check before publishing to see if any needs changing. -->
+   <!-- ADD: Write more material. -->
+ 
+ <!-- Index Entity Declarations -->
+ <!ENTITY genindex.sgm SYSTEM "genindex.sgm">
+ 
+ <!-- General Entity Declarations -->
+ 
+ <!ENTITY book "book">
+   <!-- Produce a notation for the book/manual/report/WWW page.  -->
+   <!-- Modify this to the desired noun. -->
+ <!ENTITY Book "Book">
+   <!-- Produce a capitalized version of &book;  -->
+   <!-- Modify this to the desired noun. -->
+ <!ENTITY c "<application class='software'>C</application>">
+   <!-- Produce a notation for the C programming language.  -->
+   <!-- Modify this to the desired formatting. -->
+ <!ENTITY cc "<application class='software'>C++</application>">
+   <!-- Produce a notation for the C++ programming language.  -->
+   <!-- Modify this to the desired formatting. -->
+ <!ENTITY cheetah "<application class='software'>Cheetah</application>" >
+   <!-- Produce a notation for the Cheetah Library.  -->
+ <!ENTITY dashdash "-&hairsp;-" >
+   <!-- Produce a notation for a double dash.  Without this, TeX produces an en-hyphen. -->
+ <!ENTITY doof2d "<command>Doof2d</command>" >
+   <!-- Produce a notation for the Doof2d program.  -->
+ <!ENTITY make "<application class='software'>Make</application>">
+   <!-- Produce a notation for the GNU Make program.  -->
+ <!ENTITY mm "<application class='software'>MM</application>">
+   <!-- Produce a notation for the MM Library.  -->
+ <!ENTITY mpi "<application class='software'>MPI</application>">
+   <!-- Produce a notation for the MPI package.  -->
+ <!ENTITY pdt "<application class='software'>PDToolkit</application>">
+   <!-- Produce a notation for the PDT software package.  -->
+ <!ENTITY pete "<application class='software'>PETE</application>">
+   <!-- Produce a notation for the PETE library.  -->
+ <!ENTITY pooma "<application class='software'>POOMA</application>">
+   <!-- Produce a notation for Pooma software.  -->
+ <!ENTITY poomaToolkit "<application class='software'>POOMA Toolkit</application>">
+   <!-- Produce a notation for the Pooma toolkit.  -->
+ <!ENTITY purify "<application class='software'>Purify</application>">
+   <!-- Produce a notation for the Purify library.  -->
+ <!ENTITY smarts "<application class='software'>Smarts</application>">
+   <!-- Produce a notation for the Smarts software package.  -->
+   <!-- Modify this after capitalization is decided. -->
+ <!ENTITY stl "<application class='software'>STL</application>">
+   <!-- Produce a notation for the C++ Standard Template Library software package.  -->
+ <!ENTITY tau "<application class='software'>Tau</application>">
+   <!-- Produce a notation for the Tau software package.  -->
+ 
+ <!-- Type Entity Declarations -->
+ 
+ <!ENTITY array "<type>Array</type>">
+   <!-- The "Array" type. -->
+ <!ENTITY benchmark "<type>Benchmark</type>">
+   <!-- The "Benchmark" type. -->
+ <!ENTITY brick "<type>Brick</type>">
+   <!-- The "Brick" engine type. -->
+ <!ENTITY compressiblebrick "<type>CompressibleBrick</type>">
+   <!-- The "CompressibleBrick" engine type. -->
+ <!ENTITY distributedtag "<type>DistributedTag</type>">
+   <!-- The DistributedTag Layout type. -->
+ <!ENTITY domain "<type>Domain</type>">
+   <!-- The "Domain" type. -->
+ <!ENTITY double "<type>double</type>">
+   <!-- The C "double" type. -->
+ <!ENTITY dynamicarray "<type>DynamicArray</type>">
+   <!-- The "DynamicArray" type. -->
+ <!ENTITY engine "<type>Engine</type>">
+   <!-- The "Engine" type. -->
+ <!ENTITY field "<type>Field</type>">
+   <!-- The "Field" type. -->
+ <!ENTITY inform "<type>Inform</type>">
+   <!-- The "Inform" output type. -->
+ <!ENTITY interval "<type>Interval</type>">
+   <!-- The "Interval" type. -->
+ <!ENTITY layout "<type>Layout</type>">
+   <!-- The "Layout" type without template parameters. -->
+ <!ENTITY leaffunctor "<type>LeafFunctor</type>">
+   <!-- The "LeafFunctor" type. -->
+ <!ENTITY matrix "<type>TinyMatrix</type>">
+   <!-- The Pooma matrix type. -->
+ <!ENTITY multipatch "<type>MultiPatch</type>">
+   <!-- The "MultiPatch" engine without template parameters. -->
+ <!ENTITY replicatedtag "<type>ReplicatedTag</type>">
+   <!-- The ReplicatedTag Layout type. -->
+ <!ENTITY stencil "<type>Stencil</type>">
+   <!-- The "Stencil" type. -->
+ <!ENTITY tensor "<type>Tensor</type>">
+   <!-- The Pooma tensor type. -->
+ <!ENTITY vector "<type>Vector</type>">
+   <!-- The "Vector" type. -->
+ 
+ <!-- Mathematical Entity Declarations -->
+ 
+ <!ENTITY container "C">
+   <!-- an abbreviation for a canonical container -->
+ <!ENTITY n "n">
+   <!-- the size of one dimension of an array -->
+ <!ENTITY space "&real;<superscript>d</superscript>">
+   <!-- a notation for multi-dimensional space -->
+ 
+ <!-- System and Operating System Entity Declarations -->
+ <!ENTITY gcc "<application>g++</application>">
+   <!-- The GNU Compiler Collection C++ compiler. -->
+ <!ENTITY kcc "<application>KCC</application>">
+   <!-- The KAI C++ compiler. -->
+ <!ENTITY linux "<application>Linux</application>">
+   <!-- The Linux operating system. -->
+ 
+ <!-- &pooma; URLs and Files -->
+ 
+ <!ENTITY poomaDownloadPage '<ulink url="http://pooma.codesourcery.com/pooma/download">http://pooma.codesourcery.com/pooma/download</ulink>'>
+   <!-- The WWW page supporting downloading the &pooma; source code. -->
+   <!-- UPDATE this URL. -->
+ <!ENTITY poomaHomePage '<ulink url="http://www.pooma.com/">http://www.pooma.com/</ulink>'>
+   <!-- The canonical Pooma home page. -->
+   <!-- UPDATE this filename. -->
+ <!ENTITY poomaSource "pooma-2.3.0">
+   <!-- The Pooma source code directory. -->
+ <!ENTITY poomaSourceFile "&poomaSource;.tgz">
+   <!-- The Pooma source code archive. -->
+ <!ENTITY poomaExampleDirectory "examples/Manual">
+   <!-- The directory holding this manual's example codes. -->
+ 
+ <!-- Spelling and Formatting Decisions -->
+ <!ENTITY author "author">
+   <!-- A word describing an author xor authors. -->
+   <!-- spelling: dependence, not dependency -->
+   <!-- spelling: element-wise, not elementwise -->
+   <!-- phrase: function object, not functor -->
+   <!-- spelling: multidimensional, not multi-dimensional -->
+   <!-- spelling: multiprocessor, not multi-processor -->
+   <!-- spelling: nonzero, not non-zero -->
+ 
+ <!-- External File Entities -->
+ <!ENTITY doof2d-c-element SYSTEM "./programs/Doof2d-C-element-annotated.cpp">
+   <!-- hand-coded Doof2d implementation -->
+ <!ENTITY doof2d-array-element SYSTEM "./programs/Doof2d-Array-element-annotated.cpp">
+   <!-- Array element-wise Doof2d implementation -->
+ <!ENTITY doof2d-array-parallel SYSTEM "./programs/Doof2d-Array-parallel-annotated.cpp">
+   <!-- Array data-parallel Doof2d implementation -->
+ <!ENTITY doof2d-array-stencil SYSTEM "./programs/Doof2d-Array-stencil-annotated.cpp">
+   <!-- Array stencil Doof2d implementation -->
+ <!ENTITY doof2d-array-distributed SYSTEM "./programs/Doof2d-Array-distributed-annotated.cpp">
+   <!-- distributed Array stencil Doof2d implementation -->
+ <!ENTITY doof2d-field-parallel SYSTEM "./programs/Doof2d-Field-parallel-annotated.cpp">
+   <!-- Field data-parallel Doof2d implementation -->
+ <!ENTITY doof2d-field-distributed SYSTEM "./programs/Doof2d-Field-distributed-annotated.cpp">
+   <!-- Field data-parallel distributed Doof2d implementation -->
+ <!ENTITY glossary-chapter SYSTEM "glossary.xml">
+   <!-- glossary -->
+ <!ENTITY tutorial-chapter SYSTEM "tutorial.xml">
+   <!-- Doof2d tutorial programs chapter -->
+ ]>
+ 
+ <book>
+  <bookinfo>
+   <title>&pooma;</title>
+   <subtitle>A &cc; Toolkit for High-Performance Parallel Scientific Computing</subtitle>
+   <author><firstname>Jeffrey</firstname><othername
+   role='mi'>D.</othername><surname>Oldham</surname>
+    <affiliation>
+     <orgname>CodeSourcery, LLC</orgname>
+    </affiliation>
+   </author>
+   <copyright><year>2001</year><holder>CodeSourcery, LLC (<ulink url="http://www.codesourcery.com"></ulink>)</holder></copyright>
+   <contractsponsor>Los Alamos National Laboratory<ulink url="http://www.lanl.gov"></ulink></contractsponsor>
+   <legalnotice>
+    <!-- FIXME: What is the correct legal notice? -->
+    <para>All rights reserved.  This document may not be redistributed in any form without the express permission of the author.</para>
+   </legalnotice>
+   <revhistory>
+    <revision>
+     <revnumber>0.01</revnumber>
+     <date>2001 Nov 26</date>
+     <authorinitials>jdo</authorinitials>
+     <revremark>first draft</revremark>
+    </revision>
+   </revhistory>
+  </bookinfo>
+ 
+  <!-- FINISH: May we have a short table of contents followed by a -->
+  <!-- complete table of contents? -->
+ 
+  <preface id="preface">
+   <title>Preface</title>
+ 
+   <para>FINISH: Describe the target audience for &pooma; programs and
+   for this manual: &cc; programmers writing scientific code, possibly
+   parallel execution.</para>
+ 
+   <para>Assume familiarity with &cc; template programming and the
+   standard template library.  FIXME: Remove this index
+   entry.<indexterm id="oldham"><primary>Oldham,
+   Jeffrey&nbsp;D.</primary></indexterm></para>
+ 
+   <section id="preface-notation">
+    <title>Notation</title>
+ 
+    <para>UNFINISHED</para>
+   </section>
+ 
+ 
+   <section id="preface-reading_book:">
+    <title>How to Read This &Book;</title>
+ 
+    <para>FINISH: Write this section in a style similar to Lamport's
+    LaTeX section 1.2.  FINISH: Fix the book title and the section
+    number.</para>
+   </section>
+ 
+ 
+   <section id="preface-downloading">
+    <title>Obtaining &pooma; and Sample Programs</title>
+ 
+    <para>Available for free from what WWW site?  Include what portions
+    of <filename class="libraryfile">LICENSE</filename>?  Be sure to
+    include CVS instructions as well.</para>
+ 
+    <para>Which additional packages are necessary and when?</para>
+ 
+   </section>
+ 
+ 
+   <section id="preface-using_modifying">
+    <title>Using and Modifying &pooma;</title>
+ 
+    <para>&pooma; is available under open source license.  It can be
+    used and modified by anyone, anywhere.  Can it be sold?  Include
+    <filename class="libraryfile">LICENSE</filename>.</para>
+ 
+    <para>QUESTION: How do developers contribute code?</para>
+ 
+   </section>
+ 
+ 
+   <section id="preface-acknowledgements">
+    <title>Acknowledgements</title>
+ 
+    <para>Mark Mitchell, Stephen Smith</para>
+   </section>
+ 
+  </preface>
+ 
+ 
+  <part id="programming">
+   <title>Programming with &pooma;</title>
+ 
+   <chapter id="introduction">
+    <title>Introduction</title>
+ 
+    <para>QUESTION: Add a partintro to the part above?</para>
+ 
+    <para>&pooma; abbreviates <quote>Parallel Object-Oriented Methods
+    and Application</quote>.</para>
+ 
+    <para>This document is an introduction to &pooma; v2.1, a &cc;
+    toolkit for high-performance scientific computation.  &pooma;
+    runs efficiently on single-processor desktop machines,
+    shared-memory multiprocessors, and parallel supercomputers
+    containing dozens or hundreds of processors. What's more, by making
+    extensive use of the advanced features of the ANSI/ISO &cc;
+    standard&mdash;particularly templates&mdash;&pooma; presents a
+    compact, easy-to-read interface to its users.</para>
+ 
+    <para>From Section&nbsp; of
+    <filename>papers/iscope98.pdf</filename>:</para>
+ 
+    <para>Scientific software developers have struggled with the need
+    to express mathematical abstractions in an elegant and maintainable
+    way without sacrificing performance.  The &pooma; (Parallel
+    Object-Oriented Methods and Applications) framework, written in
+    <acronym>ANSI</acronym>/<acronym>ISO</acronym> &cc;, has
+    demonstrated both high expressiveness and high performance for
+    large-scale scientific applications on platforms ranging from
+    workstations to massively parallel supercomputers.  &pooma; provides
+    high-level abstractions for multidimensional arrays, physical
+    meshes, mathematical fields, and sets of particles.  &pooma; also
+    exploits techniques such as expression templates to optimize serial
+    performance while encapsulating the details of parallel
+    communication and supporting block-based data compression.
+    Consequently, scientists can quickly assemble parallel simulation
+    codes by focusing directly on the physical abstractions relevant to
+    the system under study and not the technical difficulties of
+    parallel communication and machine-specific optimization.</para>
+ 
+    <para>ADD: diagram of science and &pooma;.  See the diagram that
+    Mark and I wrote.</para>
+ 
+    <para>Mention efficient evaluation of &pooma; expressions.  See
+    <filename>pooma-publications/iscope98.pdf</filename>,
+    Section&nbsp;4.</para>
+ 
+    <section id="introduction-pooma_evolution">
+     <title>Evolution of &pooma;</title>
+ 
+     <para>QUESTION: Is this interesting?  Even if it is, it should be
+     short.</para>
+ 
+     <para>The file <filename>papers/SCPaper-95.html</filename>
+     describes ?&pooma;1? and its abstraction layers.</para>
+ 
+     <para>The "Introduction" of
+     <filename>papers/Siam0098.ps</filename> describes the DoE's
+     funding motivation for &pooma;: Accelerated Strategic Computing
+     Initiative (ASCI) and Science-based Stockpile Stewardship (SBSS),
+     pp. 1&ndash;2.</para>
+ 
+     <para>See list of developers on p.&nbsp;1 of
+     <filename>papers/pooma.ps</filename>.</para>
+ 
+     <para>See list of developers on p.&nbsp;1 of
+     <filename>papers/pooma.ps</filename>.  See history and motivation
+     on p.&nbsp;3 of <filename>papers/pooma.ps</filename>.</para>
+ 
+     <para>Use <filename class="libraryfile">README</filename> for
+     information.</para>
+ 
+     <blockquote>
+      <attribution><filename
+ 			    class="libraryfile">introduction.html</filename></attribution>
+ 
+      <para>&pooma; was designed and implemented by scientists working
+      at the Los Alamos National Laboratory's Advanced Computing
+      Laboratory. Between them, these scientists have written and tuned
+      large applications on almost every commercial and experimental
+      supercomputer built in the last two decades. As the technology
+      used in those machines migrates down into departmental computing
+      servers and desktop multiprocessors, &pooma; is a vehicle for its
+      designers' experience to migrate as well. In particular,
+      &pooma;'s authors understand how to get good performance out of
+      modern architectures, with their many processors and multi-level
+      memory hierarchies, and how to handle the subtly complex problems
+      that arise in real-world applications.</para>
+     </blockquote>
+ 
+    </section>
+ 
+   </chapter>
+ 
+   &tutorial-chapter;
+ 
+ 
+   <chapter id="concepts">
+    <title>Overview of &pooma; Concepts</title>
+ 
+    <para>FIXME: How does multi-threaded computation fit into the
+    model?</para>
+ 
+    <para>In the previous chapter, we presented several different
+    implementations of the &doof2d; simulation program.  The
+    implementations illustrate the various containers, computation
+    syntaxes, and computation environments that &pooma; supports.  In
+    this chapter, we describe the concepts associated with each of
+    these three categories.  Specific details needed by programmers are
+    deferred to later chapters.</para>
+ 
+    <table frame="none" colsep="0" rowsep="0" tocentry="1"
+      orient="port" pgwide="0" id="concepts-table">
+      <title>&pooma; Implementation Concepts</title>
+      <tgroup cols="3" align="center">
+       <thead>
+        <row>
+         <entry>Container</entry>
+ 	<entry>Computation Syntax</entry>
+ 	<entry>Computation Environment</entry>
+        </row>
+       </thead>
+       <tbody>
+        <row>
+ 	<entry>&array;</entry>
+ 	<entry>element-wise</entry>
+ 	<entry>sequential</entry>
+        </row>
+        <row>
+ 	<entry>&dynamicarray;</entry>
+ 	<entry>data-parallel</entry>
+ 	<entry>distributed</entry>
+        </row>
+        <row>
+ 	<entry>&field;</entry>
+ 	<entry>stencil-based</entry>
+ 	<entry></entry>
+        </row>
+        <row>
+ 	<entry>&tensor;</entry>
+ 	<entry>relational</entry>
+ 	<entry></entry>
+        </row>
+        <row>
+ 	<entry>&matrix;</entry>
+ 	<entry></entry>
+ 	<entry></entry>
+        </row>
+        <row>
+ 	<entry>&vector;</entry>
+ 	<entry></entry>
+ 	<entry></entry>
+        </row>
+       </tbody>
+      </tgroup>
+     </table>
+ 
+    <para>The most important &pooma; concepts can be grouped into three
+    separate categories:
+    <variablelist>
+     <varlistentry>
+       <term>container</term>
+       <listitem>
+        <para>data structure holding one or more values and addressed
+        by indices</para>
+       </listitem>
+      </varlistentry>
+      <varlistentry>
+       <term>computation syntax</term>
+       <listitem>
+        <para>styles of expressing computations</para>
+       </listitem>
+      </varlistentry>
+      <varlistentry>
+       <term>computation environment</term>
+       <listitem>
+        <para>description of resources for computing, e.g., single
+        processor or multi-processor</para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+     See <xref linkend="concepts-table"></xref>.  Many &pooma; programs
+     select one possibility from each column.  For example, <xref
+     linkend="tutorial-array_stencil-doof2d"></xref> used a &array;
+     container and stencils for sequential computation, while <xref
+     linkend="tutorial-field_distributed-doof2d"></xref> used a &field;
+     container and data-parallel statements with distributed
+     computation.  A program may use multiple containers and various
+     computation syntax, but the computation environment either has
+     distributed processors or not.</para>
+ 
+    <para>In the rest of this chapter, we explore these three
+    categories.  First, we describe &pooma; containers, illustrating
+    the purposes of each, and explaining the concepts needed to declare
+    them.  Then, we describe the different computation syntaxes and
+    finally distributed computation concepts.</para>
+ 
+ 
+    <section id="concepts-containers">
+     <title>&pooma; Containers</title>
+ 
+     <para>Most &pooma; programs use <firstterm>containers</firstterm>
+     to store groups of values.  &pooma; containers are objects that
+     store other objects.  They control allocation and deallocation of
+     and access to these objects.  They are a generalization of &c;
+     arrays, but &pooma; containers are first-class objects so they can
+     be used directly in expressions.  They are similar to &cc;
+     containers such as <type>vector</type>, <type>list</type>, and
+     <type>stack</type>.  See <xref
+     linkend="concepts-containers-table"></xref> for a summary of the
+     containers.</para>
+ 
+     <para>This chapter describes many concepts, not all of which are
+     needed to begin programming with the &pooma; Toolkit.  Below we
+     introduce the different categories of concepts.  After that, we
+     introduce the different &pooma;'s containers and describe how to
+     choose the appropriate one for a particular task.  <xref
+     linkend="concepts-sequential_containers-declarations-dependences"></xref>
+     indicates which concepts must be understood when declaring a
+     particular container.  All of these concepts are described in
+     <xref
+     linkend="concepts-containers-sequential_declarations"></xref> and
+     <xref
+     linkend="concepts-containers-distributed_declarations"></xref>.
+     Use this figure to decide which concepts in the former are
+     relevant.  Reading the latter section is necessary only if
+     computing using multiple processors.  The programs in the previous
+     chapter illustrate many of these concepts.</para>
+ 
+     <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	   orient="port" pgwide="0" id="concepts-containers-table">
+      <title>&pooma; Container Summary</title>
+      <tgroup cols="2" align="left">
+       <tbody>
+        <row>
+ 	<entry><glossterm
+ 	linkend="glossary-array">&array;</glossterm></entry>
+ 	<entry>container mapping indices to values and that may be
+ 	used in expressions</entry>
+        </row>
+        <row>
+ 	<entry><glossterm
+ 	linkend="glossary-dynamicarray">&dynamicarray;</glossterm></entry>
+ 	<entry>one-dimensional &array; whose <glossterm
+ 	linkend="glossary-domain">domain</glossterm> can be dynamically
+ 	resized</entry>
+        </row>
+        <row>
+ 	<entry><glossterm linkend="glossary-field">&field;</glossterm></entry>
+ 	<entry>container mapping indices to one or more values and
+ 	residing in multi-dimensional space</entry>
+        </row>
+        <row>
+ 	<entry><glossterm
+ 	linkend="glossary-tensor">&tensor;</glossterm></entry>
+ 	<entry>multi-dimensional mathematical tensor</entry>
+        </row>
+        <row>
+ 	<entry><glossterm
+ 	linkend="glossary-matrix">&matrix;</glossterm></entry>
+ 	<entry>two-dimensional mathematical matrix</entry>
+        </row>
+        <row>
+ 	<entry><glossterm
+ 	linkend="glossary-vector">&vector;</glossterm></entry>
+ 	<entry>multi-dimensional mathematical vector</entry>
+        </row>
+       </tbody>
+      </tgroup>
+     </table>
+ 
+ <!-- FIXME: Want firstterm around array. -->
+ 
+     <para>A &pooma; <glossterm
+     linkend="glossary-array">array;</glossterm>, generalizing a &c;
+     array, maps indices to values.  Given a index or position in an
+     &array;'s domain, it returns the associated value, either by
+     returning a stored value or by computing it.  The use of indices,
+     which are usually ordered tuples, permits constant-time access
+     although computing a particular value may require significant
+     time.  In addition to the functionality provided by &c; arrays,
+     the &array; class automatically handles memory allocation and
+     deallocation, supports a wider variety of assignments, and can be
+     used in expressions.  For example, the addition of two arrays can
+     be assigned to an array and the product of a scalar element and an
+     array is permissible.</para>
+ 
+ <!-- FIXME: Want firstterm around dynamicarray. -->
+ 
+     <para>A &pooma; <glossterm
+     linkend="glossary-dynamicarray">&dynamicarray;</glossterm> extends
+     &array; capabilities to support a dynamically-changing domain but
+     is restricted to only one dimension.  When the &dynamicarray; is
+     resized, its values are preserved.</para>
+ 
+ <!-- FIXME: Want firstterm around field. -->
+ 
+     <para>A &pooma; <glossterm>&field;</glossterm> is an &array; with
+     spatial extent.  Each domain consists of <glossterm
+     linkend="glossary-cell"><firstterm>cell</firstterm></glossterm>s
+     in one-, two-, or three-dimensional space.  Although indexed
+     similarly to &array;s, each cell may contain multiple values and
+     multiple materials.  A &field;'s <glossterm
+     linkend="glossary-mesh">mesh</glossterm> stores its spatial
+     characteristics and can map yield, e.g., a point contained in a
+     cell, the distance between two cells, and a cell's normals.  A
+     &field; should be used whenever geometric or spatial computations
+     are needed, multiple values per index are desired, or a
+     computation involves more than one material.</para>
+ 
+ <!-- FIXME: Want firstterm around tensor. -->
+ 
+     <para>A <glossterm linkend="glossary-tensor">&tensor;</glossterm>
+     implements a multi-dimensional mathematical tensor.  Since it is a
+     first-class type, it can be used in expressions such as
+     adding two &tensor;s.</para>
+ 
+ <!-- FIXME: Want firstterm around matrix. -->
+ 
+     <para>A <glossterm linkend="glossary-matrix">&matrix;</glossterm>
+     implements a two-dimensional mathematical matrix.  Since it is a
+     first-class type, it can be used in expressions such as
+     multiplying matrices and assignments to matrices.</para>
+ 
+ <!-- FIXME: Want firstterm around vector. -->
+ 
+     <para>A <glossterm linkend="glossary-vector">&vector;</glossterm>
+     implements a multi-dimensional mathematical vector, which is an
+     ordered tuple of components.  Since it is a first-class type, it
+     can be used in expressions such as adding two &vector;s and
+     multiplying a &matrix; and a &vector;.</para>
+ 
+     <para>The data of an &array;, &dynamicarray;, or &field; can be
+     viewed using more than one container by taking a view.  A
+     <glossterm
+     linkend="glossary-view"><firstterm>view</firstterm></glossterm> of
+     an existing container&nbsp;&container; is a container whose domain
+     is a subset of&nbsp;&container;.  The subset can equal the
+     original domain.  A view acts like a reference in that changing
+     any of the view's values also changes the original container's and
+     vice versa.  While users sometimes explicitly create views, they
+     are perhaps more frequently created as temporaries in expressions.
+     For example, if <varname>A</varname> is an &array; and
+     <varname>I</varname> is a domain, <statement>A(I) -
+     A(I-1)</statement> forms the difference between adjacent
+     values.</para>
+ 
+ 
+     <section id="concepts-containers-choosing">
+      <title>Choosing a Container</title>
+ 
+      <para>The two most commonly used &pooma; containers are &array;s
+      and &field;s.  <xref
+      linkend="concepts-containers-choice_table"></xref> contains a
+      decision tree describing how to choose an appropriate
+      container.</para>
+ 
+     <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	   orient="port" pgwide="0" id="concepts-containers-choice_table">
+      <title>Choosing a &pooma; Container</title>
+      <tgroup cols="2" align="left">
+       <tbody>
+        <row>
+ 	<entry>If modeling mathematical entries,</entry>
+ 	<entry>use a &vector;, &matrix;, or &tensor;.</entry>
+        </row>
+        <row>
+ 	<entry>If indices and values reside in multi-dimensional space
+ 	&space;,</entry>
+ 	<entry>use a &field;.</entry>
+        </row>
+        <row>
+ 	<entry>If there are multiple values per index,</entry>
+ 	<entry>use a &field;.</entry>
+        </row>
+        <row>
+ 	<entry>If there are multiple materials participating in the same computation,</entry>
+ 	<entry>use a &field;.</entry>
+        </row>
+        <row>
+ 	<entry>If the domain's size dynamically changes and is one-dimensional,</entry>
+ 	<entry>use a &dynamicarray;.</entry>
+        </row>
+        <row>
+ 	<entry>Otherwise</entry>
+ 	<entry>use an &array;.</entry>
+        </row>
+       </tbody>
+      </tgroup>
+     </table>
+ 
+     </section><!-- choosing among containers -->
+ 
+ 
+     <section id="concepts-containers-sequential_declarations">
+      <title>Declaring Sequential Containers</title>
+ 
+      <figure float="1" id="concepts-sequential_containers-declarations-dependences">
+       <title>Concepts For Declaring Containers</title>
+       <mediaobject>
+        <imageobject>
+ 	<imagedata fileref="figures/concepts.111" format="EPS" align="center"></imagedata>
+        </imageobject>
+        <textobject>
+ 	<phrase>concepts involved in declaring containers</phrase>
+        </textobject>
+       </mediaobject>
+      </figure>
+ 
+      <para>In the previous sections, we introduced the &pooma;
+      containers and described how to choose one appropriate for a
+      given task.  In this section, we describe the concepts involved
+      in declaring them.  Concepts specific to distributed computation
+      are described in the next section.</para>
+ 
+      <para><xref
+      linkend="concepts-sequential_containers-declarations-dependences"></xref>
+      illustrates the containers and the concepts involved in their
+      declarations.  The containers are listed in the top row.  Lines
+      connect these containers to the components necessary for their
+      declarations.  For example, an &array; declaration requires an
+      engine and a layout.  These, in turn, depend on other &pooma;
+      concepts.  Declarations necessary only for distributed, or
+      multiprocessor, computation are surrounded by dashed lines.  You
+      can use these dependences to indicate the concepts needed for a
+      particular container.</para>
+ 
+      <para>An <glossterm
+      linkend="glossary-engine"><firstterm>engine</firstterm></glossterm>
+      stores and, if necessary, computes a container's values.  A
+      container has one or more engines.  The separation of a container
+      and its storage permits optimizing a program's space
+      requirements.  For example, a container returning the same value
+      for all indices can use a constant engine, which need only store
+      one value for the entire domain.  A &compressiblebrick; engine
+      reduces its space requirements to a constant whenever all its
+      values are the same.  The separation also permits taking <link
+      linkend="glossary-view">view</link>s of containers without
+      copying storage.</para>
+ 
+      <figure float="1" id="concepts-containers-declarations-computational_implementation">
+       <title>&array; and &field; Mathematical and Computational Concepts</title>
+       <mediaobject>
+        <imageobject>
+ 	<imagedata fileref="figures/concepts.101" format="EPS" align="center"></imagedata>
+        </imageobject>
+        <textobject>
+ 	<phrase>maps from indices to values</phrase>
+        </textobject>
+       </mediaobject>
+      </figure>
+ 
+      <para>A <glossterm
+      linkend="glossary-layout"><firstterm>layout</firstterm></glossterm>
+      maps <link linkend="glossary-domain">domain</link> indices to the
+      processors and computer memory used by a container's engines.
+      See <xref
+      linkend="concepts-containers-declarations-computational_implementation"></xref>.
+      A computer computes a container's values using a processor and
+      memory.  The layout specifies the processor(s) and memory to use
+      for each particular index.  A container's layout for a
+      uniprocessor implementation consists of its domain, the
+      processor, and its memory.  For a multi-processor implementation,
+      the layout maps portions of the domain to (possibly different)
+      processors and memory.</para>
+ 
+      <para>A &field;'s <glossterm
+      linkend="glossary-mesh"><firstterm>mesh</firstterm></glossterm>
+      maps <link linkend="glossary-domain">domain</link> indices to
+      geometric values in &space; such as distance between cells, edge
+      lengths, and normals to cells.  In other words, it provides a
+      &field;'s spatial extent.  See also <xref
+      linkend="concepts-containers-declarations-computational_implementation"></xref>.
+      Different mesh types may support different geometric
+      values.</para>
+ 
+      <para>A mesh's <glossterm
+      linkend="glossary-corner_position"><firstterm>corner
+      position</firstterm></glossterm> specifies the <link
+      linkend="glossary-point">point</link> in &space; corresponding to
+      the lower, left corner of its <link
+      linkend="glossary-domain">domain</link>.  Combining this, the
+      domain, and the cell size fully specifies the mesh's map from
+      indices to &space;.</para>
+ 
+      <para>A mesh's <glossterm
+      linkend="glossary-cell_size"><firstterm>cell
+      size</firstterm></glossterm> specifies the spatial dimensions of
+      a &field; <link linkend="glossary-cell">cell</link>, e.g., its
+      width, height, and depth, in&nbsp;&space;.  Combining this, the
+      domain, and the corner position fully specifies the mesh's map
+      from indices to &space;.</para>
+ 
+      <para>A <glossterm
+      linkend="glossary-domain"><firstterm>domain</firstterm></glossterm>
+      is a set of points on which a container can define values.  An
+      <glossterm
+      linkend="glossary-interval"><firstterm>interval</firstterm></glossterm>
+      consists of all integral points between two values.  It is
+      frequently represented using mathematical interval notation [a,b]
+      even though it contains only the integral points, e.g., a, a+1,
+      a+2, &hellip;, b.  The concept is generalized to multiple
+      dimensions by forming tensor product of intervals, i.e., all the
+      integral tuples in an &n;-dimensional space.  For example, the
+      two-dimensional containers in the previous chapter are defined on
+      a two-dimensional domain with the both dimensions' spanning the
+      interval [0,n).  A <glossterm
+      linkend="glossary-stride"><firstterm>stride</firstterm></glossterm>
+      is a subset of an interval consisting of regularly-spaced
+      points.  A <glossterm
+      linkend="glossary-range"><firstterm>range</firstterm></glossterm>
+      is a subset of an interval formed by the tensor product of strides.
+      A <glossterm
+      linkend="glossary-region"><firstterm>region</firstterm></glossterm>
+      represents a continuous &n;-dimensional domain.</para>
+     </section>
+ 
+ 
+     <section id="concepts-containers-distributed_declarations">
+      <title>Declaring Distributed Containers</title>
+ 
+      <para>In the previous section, we introduced the concepts
+      important when declaring containers for use on uniprocessor
+      computers.  When using multi-processor computers, we augment
+      these concepts with those for distributed computation.  Reading
+      this section is important only for running the same program on
+      multiple processors.  Many of these concepts were introduced in
+      <xref linkend="tutorial-array_distributed"></xref> and <xref
+      linkend="tutorial-field_distributed"></xref>.  <xref
+      linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
+      illustrates the &pooma; distributed computation model.  In this
+      section, we concentrate on the concepts necessary to declare a
+      distributed container.</para>
+ 
+      <para>As we noted in <xref
+      linkend="tutorial-array_distributed"></xref>, a &pooma;
+      programmer must specify how each container's domain should be
+      distributed among the available processors and memory spaces.
+      Using this information, the Toolkit automatically distributes the
+      data among the available processors and handles any required
+      communication among them.  The three concepts necessary for
+      declaring distributed containers are a partition, a guard layer,
+      and a context mapper tag.</para>
+ 
+      <para>A <glossterm
+      linkend="glossary-partition"><firstterm>partition</firstterm></glossterm>
+      specified how to divide a container's domain into distributed
+      pieces.  For example, the partition illustrated in <xref
+      linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
+      would divide a two-dimensional domain into three equally-sized
+      pieces along the x-dimension and two equally-sized pieces along
+      the y-dimension.  Partitions can be independent of the size of
+      container's domain.  The example partition will work on any
+      domain as long as the size of its x-dimension is a multiple of
+      three.  A domain is separated into disjoint <link
+      linkend="glossary-patch">patch</link>es.</para>
+ 
+      <para>A <glossterm
+      linkend="glossary-guard_layer"><firstterm>guard
+      layer</firstterm></glossterm> is <quote>extra</quote> domain
+      surrounding each patch.  This region has read-only values.  An
+      <glossterm
+      linkend="glossary-external_guard_layer"><firstterm>external guard
+      layer</firstterm></glossterm> specifies values surrounding the
+      domain.  Its presence eases computation along the domain's edges
+      by permitting the same computations as for more internal
+      computations.  An <glossterm
+      linkend="glossary-internal_guard_layer"><firstterm>internal guard
+      layer</firstterm></glossterm> duplicates values from adjacent
+      patches so communication with adjacent patches need not occur
+      during a patch's computation.  The use of guard layers is an
+      optimization; using external guard layers eases programming and
+      using internal guard layers reduces communication among
+      processors.  Their use is not required.</para>
+ 
+      <para>A <glossterm
+      linkend="glossary-context_mapper"><firstterm>context
+      mapper</firstterm></glossterm> indicates how a container's
+      patches are mapped to processors and shared memory.  For example,
+      the &distributedtag; indicates that the patches should be
+      distributed among the processors so each patch occurs once in the
+      entire computation.  The &replicatedtag; indicates that the
+      patches should be replicated among the processors so each
+      processing unit has its own copy of all the patches.  While it
+      could be wasteful to have different processors perform the same
+      computation, replicating a container can reduce possibly more
+      expensive communication costs.</para>
+     </section>
+ 
+ 
+     <section>
+      <title>????Computation Syntax????</title>
+ 
+      <para>UNFINISHED</para>
+     </section>
+ 
+ 
+     <section id="concepts-computation_environment">
+      <title>Computation Environment</title>
+ 
+      <para>A &pooma; program can execute on a wide variety of
+      computers.  The default <glossterm
+      linkend="glossary-sequential"><firstterm>sequential computing
+      environment</firstterm></glossterm> consists of one processor and
+      associated memory, as found on a personal computer.  In contrast,
+      a <glossterm
+      linkend="glossary-distributed"><firstterm>distributed computing
+      environment</firstterm></glossterm> may have multiple processors
+      and multiple distributed or shared memories.  For example, some
+      desktop computers have dual processors and shared memory.  A
+      large supercomputer may have thousands of processors, perhaps
+      with groups of eight sharing the same memory.</para>
+ 
+      <para>Using distributed computation requires three things: the
+      programmer must declare how container domains will be
+      distributed, &pooma; must be configured to use a communications
+      library, and the &pooma; executable must be run using the
+      library.  All of these were illustrated in <xref
+      linkend="tutorial-array_distributed"></xref> and <xref
+      linkend="tutorial-field_distributed"></xref>.  <xref
+      linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
+      illustrates the &pooma; distributed computation model.  <xref
+      linkend="concepts-containers-distributed_declarations"></xref>
+      described how to declare containers with distributed domains.
+      Detailed instructions how to configure &pooma; for distributed
+      computation appear in <xref linkend="installation"></xref>.  More
+      detailed instructions how to run distributed &pooma; executables
+      appear in <xref linkend="parallel"></xref>.  Here we present
+      three concepts for distributed computation: context, layout, and
+      a communication library.
+ 
+      A <glossterm
+      linkend="glossary-context"><firstterm>context</firstterm></glossterm>
+      is a collection of shared memory and processors that can execute
+      a program of a portion of a program.  It can have one or more
+      processors, but all these processors must access the same shared
+      memory.  Usually the computer and its operating system, not the
+      programmer, determine the available contexts.</para>
+ 
+      <para>
+ HERE</para>
+ 
+ <!-- HERE -->
+     </section>
+ 
+    </section>
+ 
+ <section>
+ <title>Extraneous Material</title>
+ 
+    <para>Describe the software application layers similar to
+    <filename>papers/SCPaper-95.html</filename> and "Short Tour of
+    &pooma;" in <filename>papers/SiamOO98_paper.ps</filename>.
+    Section&nbsp;2.2, "Why a Framework?," of
+    <filename>papers/pooma.ps</filename> argues why a layered approach
+    eases use.  Section&nbsp;3.1, "Framework Layer Description,"
+    describes the five layers.</para>
+ 
+    <para>FINISH: Write short glossary entries for each of these.</para>
+ 
+    <para>FINISH: Look through the source code to ensure all main
+    concepts are listed.</para>
+ 
+    <para>Here are (preliminary) &pooma; equations:</para>
+ 
+    <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	  orient="port" pgwide="0">
+     <title>&pooma; <quote>Equations</quote></title>
+ 
+     <tgroup cols="1" align="center">
+      <tbody>
+       <row>
+        <entry><para>field = data + materials + centering + layout + mesh</para></entry>
+        </row>
+       <row>
+        <entry><para>map from space to values</para></entry>
+       </row>
+       <row>
+        <entry><para>array = data + layout</para></entry>
+        </row>
+       <row>
+        <entry><para>map from indices to values</para></entry>
+       </row>
+       <row>
+        <entry><para>mesh = layout + origin + spacings</para></entry>
+        </row>
+       <row>
+        <entry><para>distribute domain through physical space</para></entry>
+       </row>
+       <row>
+        <entry><para>layout = domain + partition + context_mapper_tag (distributed/replicated)</para></entry>
+        </row>
+       <row>
+        <entry><para>distribute domain's blocks among processors/contexts</para></entry>
+       </row>
+       <row>
+        <entry><para>partition = blocks + guard layers</para></entry>
+        </row>
+       <row>
+        <entry><para>split domain into blocks</para></entry>
+       </row>
+       <row>
+        <entry><para>domain = newDomain</para></entry>
+        </row>
+       <row>
+        <entry><para>space of permissible indices</para></entry>
+       </row>
+      </tbody>
+     </tgroup>
+    </table>
+ 
+ 
+    <para>FINISH: Following is a first try at describing the &pooma;
+    abstraction layers.  See also paper illustration.</para>
+ 
+    <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	  orient="port" pgwide="0">
+     <title>&pooma; Abstraction Layers</title>
+ 
+     <tgroup cols="1" align="center">
+      <tbody>
+       <row>
+        <entry><para>application program</para></entry>
+       </row>
+       <row>
+        <entry><para>&array; &field; (should have
+        <type>FieldEngine</type> under it)</para></entry>
+       </row>
+       <row>
+        <entry><para>&engine;</para></entry>
+       </row>
+       <row>
+        <entry><para>evaluators</para></entry>
+       </row>
+      </tbody>
+     </tgroup>
+    </table>
+ 
+    <para>FINISH: How does parallel execution fit in?</para>
+ 
+    <para>FINISH: Should we also name and describe each layer?</para>
+    </section>
+ 
+ 
+    <section id="concepts-data_parallel">
+     <title>Data-Parallel Statements</title>
+ 
+     <para>Can we use "An Overview of &pete;" from
+     <filename>papers/PETE_DDJ/ddj_article.html</filename> or is this
+     too low-level?</para>
+ 
+     <para>Section&nbsp;3.2.1 of <filename>papers/pooma.ps</filename>
+     gives a simple example of data-parallel expression.  It also has a
+     paragraph introducing data-parallel operations and selecting
+     subsets of domains.  Section&nbsp;3.4 describes the Chained
+     Expression Object (<acronym>CEO</acronym>), apparently a precursor
+     of &pete;.  Regardless, it provides some motivation and
+     introductory material.</para>
+ 
+     <para>From Section&nbsp;4 of
+     <filename>papers/SiamOO98_paper.ps</filename>:</para>
+ 
+     <para>This version of &pete; reduces compile time of user codes
+     and utilizes compile-time knowledge of expression &domain;s for
+     better optimization.  For example, more efficient loops for
+     evaluating an expression can be generated if &pete; knows that the
+     &domain; has unit stride in memory.</para>
+ 
+     <para>Section&nbsp;4, "Expressions and Evaluators", of
+     <filename>papers/iscope98.pdf</filename> has a good explanation of
+     &pooma;&nbsp;II's expression trees and expression engines.</para>
+ 
+     <para>COMMENT: <filename
+ 			     class="libraryfile">background.html</filename> has some related
+     &pete; material.</para>
+    </section>
+ 
+    <section>
+     <title>Containers</title>
+ 
+     <section id="concepts-containers-array">
+      <title>&array;</title>
+ 
+      <blockquote>
+       <attribution>Section&nbsp;4 "Future Improvements in
+      &pooma;&nbsp;II" of
+      <filename>papers/SiamOO98_paper.ps</filename></attribution>
+       <para>
+      An &array; can be thought of as a map from one &domain; to
+      another.&hellip; &array;s depend only on the interface of
+      &domain;s.  Thus, a subset of view of an &array; can be
+      manipulated in all the same ways as the original &array;.
+      &array;s can perform indirect addressing because the output
+      &domain; one one &array; can be used as the input &domain; of
+      another &array;.  &array;s also provide individual element
+      access.</para>
+      </blockquote>
+ 
+ 
+      <para>
+       (unformatted) From
+       <filename>papers/GenericProgramming_CSE/dubois.html</filename>:
+       The &pooma; &array; concept provides an example of how these
+       generic-programming features can lead to flexible and efficient
+       code. An Array maps a fairly arbitrary input domain to an
+       arbitrary range of outputs. When used by itself, an &array;
+       object <varname>A</varname> refers to all of the values in its
+       domain. Element-wise mathematical operations or functions can be
+       applied to an array using straightforward notation, like A + B
+       or sin(A). Expressions involving Array objects are themselves
+       Arrays. The operation A(d), where d is a domain object that
+       describes a subset of A's domain, creates a view of A that
+       refers to that subset of points. Like an array expression, a
+       view is also an Array. If d represents a single point in the
+       domain, this indexing operation returns a single value from the
+       range. Equivalently, one can index an N-dimensional Array by
+       specifying N indices, which collectively specify a single point
+       in the input domain: A(i1, i2, ..., iN).</para>
+ 
+      <para>The &pooma; multi-dimensional Array concept is similar to
+       the Fortran 90 array facility, but extends it in several
+       ways. Both &pooma; and Fortran arrays can have up to seven
+       dimensions, and can serve as containers for arbitrary
+       types. Both support the notion of views of a portion of the
+       array, known as array sections in F90. The &pooma; Array concept
+       supports more complex domains, including bounded, continuous
+       (floating-point) domains. Furthermore, Array indexing in &pooma;
+       is polymorphic; that is, the indexing operation X(i1,i2) can
+       perform the mapping from domain to range in a variety of ways,
+       depending on the particular type of the Array being
+       indexed.</para>
+ 
+      <para>Fortran arrays are dense and the elements are arranged
+       according to column-major conventions. Therefore, X(i1,i2)
+       refers to element number i1-1+(i2-1)*numberRowsInA. However, as
+       Fig. 1 shows, Fortran-style "Brick" storage is not the only
+       storage format of interest to scientific programmers. For
+       compatibility with C conventions, one might want to use an array
+       featuring dense, row-major storage (a C-style Brick). To save
+       memory, it might be advantageous to use an array that only
+       stores a single value if all its element values are the
+       same. Other sparse storage schemes that only store certain
+       values may also be desirable. To exploit parallelism, it is
+       convenient for an array's storage to be broken up into patches,
+       which can be processed independently by different CPUs. Finally,
+       one can imagine an array with no data at all. For example, the
+       values can be computed from an expression involving other
+       arrays, or analytically from the indices.
+      </para>
+ 
+      <para>The &pooma; &array; Class Template</para>
+ 
+     <para>Next we describe &pooma;'s model of the Array concept, the
+      Array class template. The three most important requirements from
+      the point of view of overall design are: (1) arbitrary domain,
+      (2) arbitrary range, and (3) polymorphic indexing. These express
+      themselves in the template parameters for the &pooma; Array
+      class. The template
+      <programlisting>
+      template &lt;int Dim, class T = double, class EngineTag = Brick&gt;
+      class Array;
+      </programlisting>
+      is a specification for creating a set of classes all named
+      Array. The template parameters Dim, T, and EngineTag determine
+      the precise type of the Array. Dim represents the dimension of
+      the array's domain. T gives the type of array elements, thereby
+      defining the output range of the array. EngineTag specifies the
+      the manner of indexing and types of the indices.</para>
+ 
+      <para>End From
+      <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
+ 
+      <para>Section&nbsp;2, "Arrays and Engines," of
+      <filename>papers/iscope98.pdf</filename> describes both &array;s
+      and &engine;s.  This may or may not duplicate the material in
+      <filename>papers/GenericProgramming_CSE/dubois.html</filename>.</para>
+ 
+      <section id="concepts-containers-array-views">
+       <title>Views of &array;s</title>
+ 
+       <para>Section&nbsp;3, "Domains and Views," of
+       <filename>papers/iscope98.pdf</filename> motivates the need for
+       views:
+        <blockquote>
+ 	<para>One of the primary uses of domains is to specify
+ 	subsections of &array; objects.  Subarrays are a common
+ 	feature of array classes; however, it is often difficult to
+ 	make such subarrays behave like first-class objects.  The
+ 	&pooma;&nbsp;II engine concept provides a clean solution to
+ 	this problem: subsetting an &array; with a domain object
+ 	creates a new &array; that has a view engine.</para>
+        </blockquote>
+       </para>
+      </section>
+     </section>
+ 
+     <section id="concepts-containers-field">
+      <title>&field;</title>
+ 
+      <para>QUESTION: Do we include boundary conditions here?
+ 
+      FINISH: Do we have an example that shows something not possible
+      with &array;?</para>
+ 
+      <para>Describe and illustrate multi-material and
+      multivalue?</para>
+ 
+      <para>ADD: description of meshes and guard layers.</para>
+ 
+     </section>
+ 
+ 
+    </section>
+ 
+    <section id="concepts-engines">
+     <title>Engines</title>
+ 
+     <para>(unformatted) From
+     <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
+ 
+     <para>The Engine Concept</para>
+ 
+     <para>To implement polymorphic indexing, the Array class defers
+     data storage and data lookup to an engine object. The requirements
+     that the Array template places on its engine provide the
+     definition for the Engine concept. We'll describe these by
+     examining a simplified version of the Array template, shown in
+     Fig. 2.</para>
+ 
+     <para>First, the Array class determines and exports (makes
+     Engine_t part of Array's public interface) the type of the engine
+     class that it will use:
+     <programlisting>
+     typedef Engine&lt;Dim, T, EngineTag&gt; Engine_t;
+     </programlisting>
+     This statement declares Engine_t to be an alias for the type
+     Engine&lt;Dim,T,EngineTag&gt;. This is the first requirement
+     placed on engine classes: they must be specializations of a
+     general Engine template whose template parameters are identical to
+     those of Array. Next, the Array template determines the type of
+     scalar arguments (indices) to be used in operator(), the function
+     that implements &pooma;'s Fortran-style indexing syntax X(i1,i2):
+     <programlisting>
+     typedef typename Engine_t::Index_t Index_t;
+     </programlisting>
+     This statement defines another type alias:
+     Array&lt;Dim,T,EngineTag&gt;::Index_t is simply an alias for
+     Engine_t::Index_t. Engine_t::Index_t is a qualified name, which
+     means that the type Index_t is found in the class Engine_t. This
+     is the second requirement for the Engine concept: the class
+     Engine_t must define a public type called Index_t. This line will
+     not compile if that definition is not supplied. This indirection
+     is one of the ways that &pooma; supports polymorphic indexing. If
+     the Engine works with a discrete integer domain, it defines its
+     Index_t to be an integral type. If the Engine works in a
+     continuous domain, it defines its Index_t to be a floating-point
+     type.</para>
+ 
+     <para>The data lookup is performed in the operator() function. We
+     see that Array simply passes the indices on to its engine
+     object. Thus, we have the third requirement for the Engine
+     concept: it must provide a version of operator() that takes Dim
+     values of type Index_t.</para>
+ 
+     <para>Simply passing the indices on to the engine object may seem
+     odd. After all, engine(i,j) looks like we're just indexing another
+     array. There are several advantages to this extra level of
+     indirection. The Array class is as faithful a model of the Array
+     concept as possible, while the Engine class is a low-level
+     interface to a user-defined data source. As a result, Array has a
+     wide variety of constructors for user convenience, while engines
+     have but a few. Array supports a wide variety of overloaded
+     operator() functions for view creation and indexing. Engines
+     support indexing only. Array does not have direct access to the
+     data, which is managed by the engine object. Finally, Array has a
+     wide variety of overloaded mathematical operators and functions,
+     and works with the Portable Expression Template Engine (PETE) [4]
+     to provide efficient evaluation of Array expressions. Engines have
+     no such support. In general, Array is much more complex and
+     feature-laden than Engine. This is the prime advantage of the
+     separation of interface and implementation: Array only has to be
+     implemented once by the &pooma; developers. Engines are simple
+     enough to be written by users and plugged directly into the Array
+     framework.</para>
+ 
+     <para>Figure 3 illustrates the "Brick" specialization of the
+     Engine template, which implements Fortran-style lookup into a
+     block of memory. First, there is the general Engine template,
+     which is empty as there is no default behavior for an unknown
+     EngineTag. The general template is therefore not a model for the
+     Engine concept and Array classes attempting to use it will not
+     compile. Next, there is the definition of the Brick class, a
+     policy tag whose sole purpose is to select a particular
+     specialization of the Engine template. Finally, there is the
+     partial specialization of the Engine template. Examining its body,
+     we see the required Index_t typedef and the required operator(),
+     which follows the Fortran prescription for generating an offset
+     into the data block based on the row, column, and the number of
+     rows. All of the requirements are met, so the Brick-Engine class
+     is a model of the Engine concept.</para>
+ 
+     <para>End From
+     <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
+ 
+     <para>(unformatted) From
+     <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
+ 
+     <para>Compile-time Versus Run-Time Polymorphism</para>
+ 
+     <para>Encapsulating the indexing in an Engine class has important
+     advantages, both in terms of flexibility and efficiency. To
+     illustrate this point, we introduce the PolarGaussian-Engine
+     specialization in Fig. 4. This is an analytic engine that
+     calculates its values directly from its inputs. Unlike the
+     Brick-Engine, this engine is "indexed" with data of the same type
+     as its output: it maps a set of T's to a single T. Therefore, the
+     Index_t typedef selects T as the index type, as opposed to the int
+     in the Brick-Engine specialization. The operator() function also
+     differs in that it computes the return value according to an
+     analytic formula.</para>
+ 
+     <para>Both Engine&lt;Dim,T,Brick&gt; and
+     Engine&lt;Dim,T,PolarGaussian&gt; can be plugged in to an Array by
+     simply varying the Array's EngineTag. This is possible despite the
+     fact that the two classes exhibit dramatically different behavior
+     because they are both models of the Engine concept.</para>
+ 
+     <para>Notice that we have achieved polymorphic indexing without
+     the use of inheritance or virtual functions. For instance,
+     consider the following code snippet:
+     <programlisting>
+     Array<2, double, Brick> a;
+     Array<2, double, PolarGaussian> b;
+ 
+     double x = a(2, 3);     // x = a.engine.data[2 + 3 * a.engine.numRows];
+     double y = b(2.0, 3.0); // y = exp(-(2.0*2.0+3.0*3.0) / b.engine.delta);
+     </programlisting>
+     The data lookup functions for the two Arrays perform completely
+     different operations. Since this is accomplished using static
+     types, it is known as compile-time polymorphism. Moreover,
+     everything is known at compile time, so the functions are fully
+     inlined and optimized, thereby yielding code equivalent to that
+     shown in the comments above.</para>
+ 
+     <para>The flexibility and efficiency of compile-time polymorphism
+     cannot be duplicated with a run-time implementation. To illustrate
+     this point, in Fig. 5, we re-implement our Array concept using the
+     classic Envelope-Letter pattern [5], with the array class,
+     RTArray, being the envelope and the run-time-engine, RTEngine,
+     being the letter. RTArray defers data lookup to the engine object
+     by invoking the engine's functions through a pointer to the
+     RTEngine base class. Figure 6 illustrates the RTEngine base class
+     and Fig. 7 illustrates two descendants: RTBrick and
+     RTPolarGaussian.</para>
+ 
+     <para>The run-time implementation provides the same basic
+     functionality as the compile-time implementation, but it is not as
+     flexible or as efficient. It lacks flexibility in that the return
+     type of the indexing operation must be specified in the RTEngine
+     base class and in the RTArray class. Thus, in Figs. 5 and 6,we see
+     versions of RTArray::operator() and RTEngine::index functions that
+     take both int's and T's. If the programmer wants to add another
+     index-type option, these classes must be modified. This is a
+     violation of the open-closed principle proposed by Meyer
+     [6]. Also, since RTEngine descendants will usually only implement
+     one version of index, we cannot make RTEngine an abstract base
+     class. Instead, we have the default versions of index throw an
+     exception. Thus, compile-time error checking is
+     weakened. Furthermore, since indexing is done via a virtual
+     function call, it will almost never be inlined, which is not
+     acceptable in most scientific applications.</para>
+ 
+     <para>There are advantages to the Envelope-Letter approach. First,
+     all RTArray objects have the same type, allowing them to be stored
+     in homogeneous collections. This can simplify the design of some
+     applications. Second, RTArray objects can change their engines at
+     runtime, and thus effectively change their types on the fly??this
+     is the primary reason for using the Envelope-Letter idiom, and can
+     be very important in some applications.</para>
+ 
+     <para>For most scientific applications, however, these issues are
+     minor, and maximum performance for array indexing is of paramount
+     importance. Our compile-time approach achieves this performance
+     while providing the desired polymorphic indexing.</para>
+ 
+     <para>From Section&nbsp;4 of
+     <filename>papers/SiamOO98_paper.ps</filename>:</para>
+ 
+     <para>The &array; class is templated on an &engine; type that
+     handles the actual implementation of the mapping from input to
+     output.  Thus, the &array; interface features are completely
+     separate from the implementation, which could be a single &c;
+     array, a function of some kind or some other mechanism.  This
+     flexibility allows an expression itself to be viewed through the
+     &array; interface.  Thus, one can write something like
+     <programlisting>
+     foo(A*B+C);
+     </programlisting> where <varname>A</varname>, <varname>B</varname> and
+     <varname>C</varname> are &array;s and <function>foo</function> is
+     a function taking an &array; as an argument.  The expression
+     <statement><varname>A</varname>*<varname>B</varname>+<varname>C</varname></statement>
+     will only be evaluated by the expression engine as needed by
+     <function>foo</function>.</para>
+ 
+     <para>In fact, one can even write &engine;s which are wrappers
+     around external data structures created in non-&pooma; codes and
+     know to manipulate these structures.  Once this is done, the
+     external entities have access to the entire &array; interface and
+     can utilize all of the powerful features of
+     &pooma;&nbsp;II.</para>
+ 
+     <para>Section&nbsp;2, "Arrays and Engines," of
+     <filename>papers/iscope98.pdf</filename> describes both &array;s
+     and &engine;s.  This may or may not duplicate the material in
+     <filename>papers/GenericProgramming_CSE/dubois.html</filename>.</para>
+ 
+     <para>Section&nbsp;4, "Expressions and Evaluators", of
+     <filename>papers/iscope98.pdf</filename> has a good explanation of
+     &pooma;&nbsp;II's expression trees and expression engines.</para>
+ 
+     <variablelist>
+      <varlistentry><term><type>MultiPatch</type> Engine</term>
+       <listitem><para>From <filename
+       class="libraryfile">README</filename>: To actually use multiple
+       contexts effectively, you need to use the MultiPatch engine with
+       patch engines that are Remote engines.  Then the data will be
+       distributed across multiple contexts instead of being copied on
+       every context.  See the files in example/Doof2d for a simple
+       example that creates a MultiPatch array that can be distributed
+       across multiple contexts and performs a stencil computation on
+       that array.</para></listitem>
+      </varlistentry>
+     </variablelist>
+    </section>
+ 
+ 
+    <section id="concepts-relations">
+     <title>Relations</title>
+ 
+     <para>UNFINISHED</para>
+    </section>
+ 
+ 
+    <section id="concepts-stencils">
+     <title>Stencils</title>
+ 
+     <para>Section&nbsp;3.5.4, "Stencil Objects," of
+     <filename>papers/pooma.ps</filename> provides a few uses of
+     stencils.</para>
+ 
+     <para>Section&nbsp;5, "Performance," of
+     <filename>papers/iscope98.pdf</filename> motivates and explains
+     stencils.</para>
+    </section>
+ 
+ 
+    <section id="concepts-contexts">
+     <title>Contexts</title>
+ 
+     <blockquote>
+      <attribution><filename
+      class="libraryfile">background.html</filename></attribution>
+      <para>In order to be able to cope with the variations in machine
+      architecture noted above, &pooma;'s distributed execution model
+      is defined in terms of one or more contexts, each of which may
+      host one or more threads. A context is a distinct region of
+      memory in some computer. The threads associated with the context
+      can access data in that memory region and can run on the
+      processors associated with that context. Threads running in
+      different contexts cannot access memory in other contexts.</para>
+ 
+      <para>A single context may include several physical processors,
+      or just one. Conversely, different contexts do not have to be on
+      separate computers&mdash;for example, a 32-node SMP computer could
+      have up to 32 separate contexts. This release of &pooma; only
+      supports a single context for each application, but can use
+      multiple threads in the context on supported platforms. Support
+      for multiple contexts will be added in an upcoming
+      release.</para>
+     </blockquote>
+    </section>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="sequential">
+    <title>Writing Sequential Programs</title>
+ 
+    <para>&pooma; can reorder computations to permit more efficient
+    computation.  When running a sequential program, reordering may
+    permit omission of unneeded computations.  For example, if only
+    values from a particular field are printed, only computations
+    involving the field and containers dependent on it need to occur.
+    When running a distributed program, reordering may permit
+    computation and communication among processors to overlap.  &pooma;
+    automatically tracks dependences between data-parallel expressions,
+    ensuring correct ordering.  It does not track statements accessing
+    particular &array; and &field; values so the programmer must
+    precede these statements with calls to
+    <function>Pooma::blockAndEvaluate()</function>.  Each call forces
+    the executable to wait until all computation has completed.  Thus,
+    the desired values are known to be available.  In practice, some
+    calls to <function>Pooma::blockAndEvaluate</function> may not be
+    necessary, but omitting them requires knowledge of &pooma;'s
+    dependence computations, so the &author; recommends calling
+    <function>Pooma::blockAndEvaluate</function> before each access to
+    a particular value in an &array; or &field;.  Omitting a necessary
+    call may lead to a race condition.  See <xref
+    linkend="debugging_profiling-missing_blockandevaluate"></xref> for
+    instructions how to diagnose and eliminate these race conditions.</para>
+ 
+    <para>Section&nbsp;3, "Domains and Views," of
+    <filename>papers/iscope98.pdf</filename> describes five types of
+    domains.</para>
+ 
+    <para>UNFINISHED</para>
+ 
+    <section id="sequential-benchmarks">
+     <title>&benchmark; Programs</title>
+ 
+     <para>Define a &benchmark; program vs. an example or an
+     executable.  Provide a short overview of how to run these
+     programs.  Provide an overview of how to write these programs.
+     See <filename
+     class="headerfile">src/Utilities/Benchmark.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="sequential-inform">
+     <title>Using <type>Inform</type>s for Output</title>
+ 
+     <para>UNFINISHED</para>
+    </section>
+ 
+ 
+    <section>
+     <title>Miscellaneous</title>
+ 
+    <para>Section&nbsp;3, "Domains and Views," of
+     <filename>papers/iscope98.pdf</filename> describes five types of
+     domains.</para>
+    </section>
+   </chapter>
+ 
+ 
+   <chapter id="parallel">
+    <title>Writing Distributed Programs</title>
+ 
+    <para>Discuss the distributed model and guard cells.  See <filename
+    class="libraryfile">docs/parallelism.html</filename>.</para>
+ 
+    <para>Does any of the parallel implementation described in
+    <filename>papers/SCPaper-95.html</filename> still apply?</para>
+ 
+    <para>?Tuning program for maximize parallel performance?</para>
+ 
+    <para>external references to &mpi; and threads</para>
+ 
+    <para>QUESTION: Are there interesting, short parallel programs in
+    any &mpi; book that we can convert to &pooma;?</para>
+ 
+    <section>
+     <title>Layouts</title>
+ 
+     <para>An out-of-date description can be found in Section&nbsp;3.3,
+     especially 3.3.2, of <filename>papers/pooma.ps</filename>
+     describes the global/local interactions and parallel abstraction
+     layers.</para>
+    </section>
+ 
+    <section>
+     <title>Parallel Communication</title>
+ 
+     <para>An out-of-date description can be found in
+      Section&nbsp;3.3.3 of <filename>papers/pooma.ps</filename></para>
+    </section>
+ 
+    <section>
+     <title>Using Threads</title>
+ 
+     <para>QUESTION: Where do threads fit into the manual?  Do threads
+     even work?</para>
+ 
+     <para>From Section&nbsp;4, of
+     <filename>papers/SiamOO98_paper.ps</filename></para>
+ 
+     <para>&pooma;&nbsp;II will make use of a new parallel run-time
+     system called &smarts; that is under development at the ACL.
+     &smarts; supports lightweight threads, so the evaluator will be
+     able to farm out data communication tasks and the evaluation of
+     subsets of an expression to multiple threads, thus increasing the
+     overlap of communication and computation.  Threads will also be
+     available at the user level for situations in which a
+     task-parallel approach is deemed appropriate.</para>
+    </section>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="debugging_profiling">
+    <title>Debugging and Profiling &pooma; Programs</title>
+ 
+    <para>Consider &dashdash;pooma-debug <replaceable>number</replaceable>.
+    See also other &pooma; options in <filename
+    class="headerfile">src/Utilities/Options.h</filename>.</para>
+ 
+    <para>UNFINISHED</para>
+     <section id="debugging_profiling-missing_blockandevaluate">
+      <title>Finding Race Conditions From Missing
+      <function>blockAndEvaluate</function> Calls</title>
+ 
+       <para>&pooma; may reorder computations so calls to
+       <function>Pooma::blockAndEvaluate()</function> are necessary
+       before accessing particular &array; and &field; values.
+       Omission of necessary calls can lead to race conditions where
+       the ordering of reads and writes to particular values is
+       incorrect.  To help diagnose if calls to
+       <function>Pooma::blockAndEvaluate</function> are missing, invoke
+       a &pooma; executable with the
+       <statement>&dashdash;pooma-blocking-expressions</statement> option.
+       This automatically causes
+       <function>Pooma::blockAndEvaluate</function> to be called after
+       each statement.  Doing so ensures program correctness, but it
+       may increase running times, particularly if multiple processors
+       are used, because computation and communication may not overlap
+       as much as possible.  Of course, program correctness is more
+       important than execution speed.</para>
+ 
+       <para>If using
+       <statement>&dashdash;pooma-blocking-expressions</statement> changes a
+       program's output, it is missing one or more calls to
+       <function>Pooma::blockAndEvaluate</function>.  To narrow the
+       region with a missing call, surround the region in question with
+       calls to <statement>Pooma::blockingExpressions(true)</statement>
+       and <statement>Pooma::blockingExpressions(false)</statement>,
+       but do not use the
+       <statement>&dashdash;pooma-blocking-expressions</statement> option.
+       Within the region, <function>Pooma::blockAndEvaluate</function>
+       will be invoked after each statement.  Repeatedly reducing the
+       region's size should reveal where calls are missing.</para>
+     </section>
+   </chapter>
+ 
+  </part>
+ 
+  <part id="reference">
+   <title>&pooma; Reference Manual</title>
+ 
+   <chapter id="tmp_ref">
+    <title>TMP: This Chapter Holds These Comments But Will Be Removed</title>
+ 
+    <para>For each template parameter need to describe the constraints
+    on it.</para>
+ 
+    <para>Remove this section when the following concerns have been
+    addressed.</para>
+ 
+    <para>Add a partintro explaining file suffixes such as <filename
+    class="headerfile">.h</filename>, <filename
+    class="libraryfile">.cpp</filename>, <filename
+    class="libraryfile">.cmpl.cpp</filename>, <filename
+    class="libraryfile">.mk</filename>, <filename
+    class="libraryfile">.conf</filename>.  Should we also explain use
+    of <literal>inline</literal> even when necessary and the template
+    model, <!-- FIXME: s/literal/keyword/ --> e.g., including <filename
+    class="libraryfile">.cpp</filename> files.</para>
+ 
+    <para>QUESTION: What are the key concepts around which to organize
+    the manual?</para>
+ 
+    <para>QUESTION: What format should the manual use?</para>
+ 
+    <blockquote>
+     <attribution>Musser, Derge, and Sanai, &sect;20.0.</attribution>
+     <para>It is important to state the requirements on the components
+     as generally as possible.  For example, instead of saying
+     <quote>class <type>X</type> must define a member function
+     <function>operator++()</function>,</quote> we say <quote>for any
+     object&nbsp;<varname>x</varname> of type&nbsp;<type>X</type>,
+     <classname>++x</classname> is defined.</quote></para>
+    </blockquote>
+   </chapter>
+ 
+ 
+   <chapter id="class_structure_ref">
+    <title>A Typical &pooma; Class</title>
+ 
+    <variablelist>
+     <title>Class Member Notation</title>
+     <varlistentry>
+      <term><fieldsynopsis>
+        <varname>*_t</varname>
+       </fieldsynopsis>
+      </term>
+      <listitem>
+       <para>type within a class.  QUESTION: What is the &cc; name for
+       this?</para>
+      </listitem>
+     </varlistentry>
+     <varlistentry>
+      <term><fieldsynopsis>
+        <varname>*_m</varname>
+       </fieldsynopsis>
+      </term>
+      <listitem>
+       <para>data member</para>
+      </listitem>
+     </varlistentry>
+    </variablelist>
+ 
+    <variablelist>
+     <title>&pooma; Class Vocabulary</title>
+     <varlistentry>
+      <term>component</term>
+      <listitem>
+       <para>one of several values packaged together.  For example, a
+       three-dimensional vector has three components, i.e., three
+       values.</para>
+      </listitem>
+     </varlistentry>
+     <varlistentry>
+      <term>element-wise</term>
+      <listitem>
+       <para>applied to each element in the group, e.g., an array</para>
+      </listitem>
+     </varlistentry>
+     <varlistentry>
+      <term>reduction</term>
+      <listitem>
+       <para>repeated application of a binary operator to all elements,
+       yielding one value</para>
+      </listitem>
+     </varlistentry>
+     <varlistentry>
+      <term>tag</term>
+      <listitem>
+       <para>an enumerated value indicating inclusion in a particular
+       semantic class.  The set of values need not be explicitly
+       declared.</para>
+      </listitem>
+     </varlistentry>
+    </variablelist>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="installing_configuring_ref">
+    <title>Installing and Configuring &pooma;</title>
+ 
+    <orderedlist>
+     <listitem>
+      <para>Installing &pooma;.</para>
+     </listitem>
+     <listitem>
+      <para>Requirements for configuration files.</para>
+     </listitem>
+    </orderedlist>
+ 
+    <para>Include descriptions of using &smarts;, &cheetah;, &tau;,
+    &pdt;.</para>
+ 
+    <para>QUESTION: Does it install on windows and on mac?  If so, what
+    are the instructions?  See also <filename
+    class="libraryfile">INSTALL.{mac,unix,windows}.</filename></para>
+ 
+    <para><filename class="libraryfile">README</filename> has some
+    information on &cheetah; and threads in the <quote>Message-Based
+    Parallelism</quote> section.</para>
+ 
+    <para>Which additional packages are necessary and when?</para>
+ 
+    <para>What configure options should we list?  See <filename
+    class="libraryfile">configure</filename>.  Be sure to list
+    debugging option and how its output relates to <filename
+    class="libraryfile">config/LINUXgcc.suite.mk</filename>.</para>
+ 
+    <para><filename class="directory">config/arch</filename> has files
+    for (OS, compiler) pairs.  Explain how to modify a configuration
+    file.  List requirements when making a new configuration file (low
+    priority).</para>
+ 
+    <para><filename
+    class="libraryfile">config/LINUXgcc.suite.mk</filename> has output
+    from <filename class="libraryfile">configure</filename>.  Useful to
+    relate to configuration files and <filename
+    class="libraryfile">configure</filename>'s debugging output.</para>
+    
+   </chapter>
+ 
+ 
+   <chapter id="compilation_ref">
+    <title>Compilation and &make; Files</title>
+ 
+    <para>We assume Gnu make.  Do we know what assumptions are made?</para>
+ 
+    <para>How do all these files interact with each other?  Ala a make
+    interpreter, give an example of which files are read and
+    when.</para>
+ 
+    <variablelist>
+     <varlistentry><term><filename
+     class="libraryfile">config/Shared/README.make</filename></term>
+     <listitem><para>This has short descriptions of many files,
+     especially in <filename
+     class="directory">config/Shared</filename>.</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">makefile</filename></term>
+     <listitem><para>These appear throughout all directories.  What are
+     the equivalences classes and what are their
+     parts?</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">include.mk</filename></term>
+     <listitem><para>What does this do?  Occurs in many directories:
+     when?  Template seems to be <filename
+     class="libraryfile">config/Shared/include2.mk</filename>.</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">subdir.mk</filename></term>
+     <listitem><para>list of subdirectories; occurs in several
+     directories: when?  <filename
+     class="libraryfile">src/subdir.mk</filename> is a good
+     example.</para></listitem>
+     </varlistentry>
+     <varlistentry>
+      <term><filename class="libraryfile">objfile.mk</filename></term>
+      <listitem>
+       <para>list of object files to construct, presumably from
+       <filename>*.cmpl.cpp</filename> files.
+       <filename>src/Utilities/objfile.mk</filename> is an
+       example.</para>
+      </listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">config/Shared/rules.mk</filename></term>
+     <listitem><para>most compiler rules</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">config/head.mk</filename></term>
+     <listitem><para>read at beginning of each
+     makefile?</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">config/Shared/tail.mk</filename></term>
+     <listitem><para>read at end of each makefile?</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">config/Shared/variables.mk</filename></term>
+     <listitem><para>Is this used?</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">config/Shared/compilerules.mk</filename></term>
+     <listitem><para>table of origin and target suffixes and commands
+     for conversion</para></listitem>
+     </varlistentry>
+ 
+    </variablelist>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="arrays_ref">
+    <title>&array;s</title>
+ 
+    <para>Include <filename
+ 			   class="headerfile">src/Pooma/Arrays.h</filename> to use &array;s.
+    The implementation source code is in <filename
+ 						  class="directory">src/Array</filename>.</para>
+ 
+    <para>FINISH: Define an array.  Introduce its parts.</para>
+ 
+    <para>ADD: some mention of the maximum supported number of
+    dimensions somewhere.</para>
+ 
+    <section id="arrays_ref-array">
+     <title>The &array; Container</title>
+ 
+     <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	   orient="port" pgwide="0">
+      <title>Template Parameters</title>
+ 
+      <tgroup cols="2" align="left">
+       <thead>
+        <row>
+ 	<entry>Parameter</entry>
+ 	<entry>Interpretation</entry>
+        </row>
+       </thead>
+       <tbody>
+        <row>
+ 	<entry><varname>Dim</varname></entry>
+ 	<entry><para>dimension</para></entry>
+        </row>
+        <row>
+ 	<entry><varname>T</varname></entry>
+ 	<entry><para>array element type</para></entry>
+        </row>
+        <row>
+ 	<entry>EngineTag</entry>
+ 	<entry><para>type of computation engine object</para></entry>
+        </row>
+       </tbody>
+      </tgroup>
+     </table>
+ 
+     <para>QUESTION: How do I introduce class type definitions, when
+     they are used, i.e., compile-time or run-time, and when
+     programmers should use them?</para>
+ 
+     <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	   orient="port" pgwide="0">
+      <title>Compile-Time Types and Values</title>
+ 
+      <tgroup cols="2" align="left">
+       <thead>
+        <row>
+ 	<entry>Type or Value</entry>
+ 	<entry>Interpretation</entry>
+        </row>
+       </thead>
+       <tbody>
+        <row>
+ 	<entry><fieldsynopsis><varname>This_t</varname></fieldsynopsis></entry>
+ 	<entry><para>the &array; object's type</para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>Engine_t</varname></fieldsynopsis></entry>
+ 	<entry><para>the &array; object's engine's type</para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>EngineTag_t</varname></fieldsynopsis></entry>
+ 	<entry><para>indication of engine's category</para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>Element_t</varname></fieldsynopsis></entry>
+ 	<entry><para>the type of the array elements, i.e., <type>T</type></para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>ElementRef_t</varname></fieldsynopsis></entry>
+ 	<entry><para>the type of a reference to an array element,
+ 	i.e., <type>T&</type>.  Equivalently, the type to write to a
+ 	single element.</para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>Domain_t</varname></fieldsynopsis></entry>
+ 	<entry><para>the array's domain's type, i.e., the type of the
+ 	union of all array indices</para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>Layout_t</varname></fieldsynopsis></entry>
+ 	<entry><para>unknown</para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>dimensions</varname></fieldsynopsis></entry>
+ 	<entry><para>integer equalling the number of dimensions, i.e.,
+ 	<varname>Dim</varname></para></entry>
+        </row>
+        <row>
+ 	<entry><fieldsynopsis><varname>rank</varname></fieldsynopsis></entry>
+ 	<entry><para>integer equalling the number of dimensions, i.e.,
+ 	<varname>Dim</varname>; a synonym for <fieldsynopsis>
+ 	   <varname>dimensions</varname></fieldsynopsis></para></entry>
+        </row>
+       </tbody>
+      </tgroup>
+     </table>
+ 
+     <section id="arrays_ref-array-constructors">
+      <title>Constructors and Destructors</title>
+ 
+      <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	    orient="port" pgwide="0">
+       <title>Constructors and Destructors</title>
+ 
+       <tgroup cols="2" align="left">
+        <thead>
+ 	<row>
+ 	 <entry>Function</entry>
+ 	 <entry>Effect</entry>
+ 	</row>
+        </thead>
+        <tbody>
+ 	<row>
+ 	 <entry>
+           <constructorsynopsis>
+ 	   <methodname>Array</methodname>
+ 	   <void></void>
+ 	  </constructorsynopsis>
+          </entry>
+ 	 <entry><para>Creates an array that will be resized
+ 	 later.</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>
+           <constructorsynopsis>
+ 	   <methodname>Array</methodname>
+ 	   <methodparam><modifier>const</modifier> <type>Engine_t&</type>
+ 	    <parameter>engine</parameter></methodparam>
+ 	  </constructorsynopsis>
+          </entry>
+ 	 <entry><para>Creates an array with an engine equivalent to
+          the <parameter>engine</parameter>.  This array will have the
+          same values as <parameter>engine</parameter>.  QUESTION: Why
+          would a user every want to use this
+          constructor?</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>
+           <constructorsynopsis>
+ 	   <methodname>Array</methodname>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type> Engine&lt;Dim2, T2, EngineTag2&gt;&</type>
+ 	    <parameter>engine</parameter>
+ 	   </methodparam>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type>Initializer&</type> <parameter>init</parameter>
+ 	   </methodparam>
+ 	  </constructorsynopsis>
+          </entry>
+ 	 <entry><para>What does this do?</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>ADD ALL CONSTRUCTORS AND DESTRUCTORS.</entry>
+ 	</row>
+        </tbody>
+       </tgroup>
+      </table>
+     </section>
+ 
+ 
+     <section id="arrays_ref-array-initializers">
+      <title>Initializers</title>
+ 
+      <para>Add a table.</para>
+     </section>
+ 
+ 
+     <section id="arrays_ref-array-access">
+      <title>Element Access</title>
+ 
+      <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	    orient="port" pgwide="0">
+       <title>&array; Element Access</title>
+ 
+       <tgroup cols="2" align="left">
+        <thead>
+ 	<row>
+ 	 <entry>Function</entry>
+ 	 <entry>Effect</entry>
+ 	</row>
+        </thead>
+        <tbody>
+ 	<row>
+ 	 <entry>
+           <methodsynopsis>
+ 	   <type>Element_t</type> <methodname>read</methodname>
+ 	   <void></void>
+ 	  </methodsynopsis>
+          </entry>
+ 	 <entry><para>unknown: See line 1839.</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>
+           <methodsynopsis>
+ 	   <type>Element_t</type> <methodname>read</methodname>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type>Sub1&</type> <parameter>s1</parameter>
+ 	   </methodparam>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type> Sub2&</type> <parameter>s2</parameter>
+ 	   </methodparam>
+ 	  </methodsynopsis>
+          </entry>
+ 	 <entry><para>How does the version with template parameters,
+          e.g., <type>Sub1</type> differ from the <type>int</type>
+          version?</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>
+           <methodsynopsis>
+ 	   <type>Element_t</type> <methodname>operator()</methodname>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type>Sub1&</type> <parameter>s1</parameter>
+ 	   </methodparam>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type>Sub2&</type> <parameter>s2</parameter>
+ 	   </methodparam>
+ 	  </methodsynopsis>
+          </entry>
+ 	 <entry><para>How does this differ from <statement>read(const
+           Sub1& s1, const Sub2& s2)</statement>?  </para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>ADD ALL <function>read</function>s and
+ 	 <function>operator()</function>s.</entry>
+ 	</row>
+        </tbody>
+       </tgroup>
+      </table>
+     </section>
+ 
+ 
+     <section id="arrays_ref-array-component">
+      <title>Component Access</title>
+ 
+      <para>When an array stores elements having components, e.g., an
+      array of vectors, tensors, or arrays, the
+      <methodname>comp</methodname> returns an array consisting of the
+      specified components.  The original and component array share the
+      same engine so changing the values in one affects values in the
+      other.</para>
+ 
+      <para>For example, if &n; &times; &n; array <varname>a</varname>
+      consists of three-dimensional real-valued vectors,
+      <statement>a.comp(1)</statement> returns a &n; &times; &n;
+      real-valued array of all the middle vector components.  Assigning
+      to the component array will also modify the middle components of
+      the vectors in <varname>a</varname>.</para>
+ 
+      <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	    orient="port" pgwide="0">
+       <title>&array; Component Access</title>
+ 
+       <tgroup cols="2" align="left">
+        <thead>
+ 	<row>
+ 	 <entry>Function</entry>
+ 	 <entry>Effect</entry>
+ 	</row>
+        </thead>
+        <tbody>
+ 	<row>
+ 	 <entry>
+           <methodsynopsis>
+ 	   <type>UNKNOWN compute this</type> <methodname>comp</methodname>
+ 	   <methodparam>
+ 	    <modifier>const</modifier>
+ 	    <type>int&</type>
+ 	    <parameter>i1</parameter>
+ 	   </methodparam>
+ 	  </methodsynopsis>
+          </entry>
+ 	 <entry><para>unknown: See line 1989.</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>ADD ALL <methodname>comp</methodname>s.</entry>
+ 	</row>
+        </tbody>
+       </tgroup>
+      </table>
+     </section>
+ 
+     <section id="arrays_ref-array-accessors">
+      <title>Accessors</title>
+ 
+      <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	    orient="port" pgwide="0">
+       <title>&array; Accessor Methods</title>
+ 
+       <tgroup cols="2" align="left">
+        <thead>
+ 	<row>
+ 	 <entry>Function</entry>
+ 	 <entry>Effect</entry>
+ 	</row>
+        </thead>
+        <tbody>
+ 	<row>
+ 	 <entry>
+           <methodsynopsis>
+ 	   <type>int</type> <methodname>first</methodname>
+ 	   <methodparam>
+ 	    <type>int</type>
+ 	    <parameter>d</parameter>
+ 	   </methodparam>
+ 	  </methodsynopsis>
+          </entry>
+ 	 <entry><para>unknown: See line 2050</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>ADD ALL other accessor methods, including
+ 	 <methodname>engine</methodname>.</entry>
+ 	</row>
+        </tbody>
+       </tgroup>
+      </table>
+     </section>
+ 
+ 
+     <section id="arrays_ref-array-copying">
+      <title>Copying &array;s</title>
+ 
+      <para>Explain how copied arrays and views of arrays share the
+      same underlying engine so changing values in one also affects the
+      other.  This is called a shallow copy.</para>
+     </section>
+ 
+ 
+     <section id="arrays_ref-array-utilities">
+      <title>Utility Methods</title>
+ 
+      <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	    orient="port" pgwide="0">
+       <title>&array; Utility Methods</title>
+ 
+       <tgroup cols="2" align="left">
+        <thead>
+ 	<row>
+ 	 <entry>Function</entry>
+ 	 <entry>Effect</entry>
+ 	</row>
+        </thead>
+        <tbody>
+ 	<row>
+ 	 <entry>
+           <methodsynopsis>
+ 	   <type>void</type> <methodname>makeOwnCopy</methodname>
+ 	   <void></void>
+ 	  </methodsynopsis>
+          </entry>
+ 	 <entry><para>unknown: See line 2044</para></entry>
+ 	</row>
+ 	<row>
+ 	 <entry>ADD ALL other utility methods.</entry>
+ 	</row>
+        </tbody>
+       </tgroup>
+      </table>
+     </section>
+ 
+     
+     <section id="arrays_ref-array-implementation">
+      <title>Implementation Details</title>
+ 
+      <para>As a container, an &array;'s implementation is quite
+      simple.  Its <keywordname>private</keywordname>data consists of
+      an engine, and it has no <keywordname>private</keywordname>
+      functions.</para>
+ 
+      <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	    orient="port" pgwide="0">
+       <title>&array; Implementation Data</title>
+ 
+       <tgroup cols="2" align="left">
+        <thead>
+ 	<row>
+ 	 <entry>Data Member</entry>
+ 	 <entry>Meaning</entry>
+ 	</row>
+        </thead>
+        <tbody>
+ 	<row>
+ 	 <entry>
+           <fieldsynopsis>
+ 	   <modifier>private</modifier>
+ 	   <type>Engine_t</type> <varname>engine_m</varname>
+ 	  </fieldsynopsis>
+          </entry>
+ 	 <entry><para>engine computing the array's values</para></entry>
+ 	</row>
+        </tbody>
+       </tgroup>
+      </table>
+ 
+     </section>
+    </section>
+ 
+ 
+    <section id="arrays_ref-dynamicarray">
+     <title>&dynamicarray;s: Dynamically-Sized Domains</title>
+ 
+     <para>A DynamicArray is a read-write array with extra
+     create/destroy methods.  It can act just like a regular Array, but
+     can have a dynamically-changing domain.  See <filename
+ 							   class="headerfile">src/DynamicArray/DynamicArray.h</filename>.</para>
+ 
+     <para>ADD: Briefly describe what the class does and an example of
+     where it is used.</para>
+ 
+     <para>ADD: Check that its interface is actually the same as for
+     &array;.</para>
+ 
+     <para>ADD: Check that the operations on dynamic arrays are
+     actually the same as for &array;.  See <filename
+ 						     class="headerfile">src/DynamicArray/DynamicArrayOperators.h</filename>,
+     <filename
+ 	      class="headerfile">src/DynamicArray/PoomaDynamicArrayOperators.h</filename>,
+     and <filename
+ 		  class="headerfile">src/DynamicArray/VectorDynamicArrayOperators.h</filename>.</para>
+ 
+     
+     <section id="arrays_ref-dynamicarray-implementation">
+      <title>Implementation Details</title>
+ 
+      <para><type>DynamicArray</type> has no
+      <keywordname>protected</keywordname> or
+      <keywordname>private</keywordname> members.</para>
+     </section>
+    </section>
+ 
+ 
+    <section id="arrays_ref-views">
+     <title>Views of &array;s</title>
+ 
+     <para>UNFINISHED</para>
+    </section>
+ 
+ 
+    <section id="arrays_ref-assignment">
+     <title>&array; Assignments</title>
+ 
+     <para>&pooma; supports assignments to &array;s of other &array;s
+     and scalar values.  QUESTION: Is the following correct?  For the
+     former, the right-hand side array's domain must be at least as
+     large as the left-hand side array's domain.  Corresponding values
+     are copied.  Assigning a scalar value to an array ensures all the
+     array elements have the same scalar value.</para>
+ 
+     <para>UNFINISHED: Add a table containing assignment operators
+     found one lines 2097&ndash;2202.</para>
+    </section>
+ 
+ 
+    <section id="arrays_ref-print">
+     <title>Printing &array;s</title>
+ 
+     <para>&array;s support output to but not input from IO streams.
+     In particular, output to <type>ostream</type>s and file streams is
+     supported.</para>
+ 
+     <para>Add a table, using <filename
+     class="headerfile">src/Array/Array.h</filename>, lines
+     2408&ndash;2421.  See the implementation in <filename
+     class="headerfile">src/Array/PrintArray.h</filename>.</para>
+ 
+     <para>QUESTION: How does one print a &dynamicarray;.</para>
+    </section>
+ 
+ 
+    <section>
+     <title>Expressions Involving &array;s</title>
+ 
+     <para>In &pooma;, expressions may contain entire &array;s.  That
+     is, &array;s are first-class objects with respect to expressions.
+     For example, given &array;s <varname>a</varname> and
+     <varname>b</varname>, the expression <statement>a + b</statement>
+     is equivalent to an array containing the element-wise sum of the
+     two arrays.</para>
+ 
+     <para>Any finite number of the operators listed below can be used
+     in an expression.  The precedence and order of operation is the
+     same as with ordinary built-in types.</para>
+ 
+     <para>QUESTION: Do &field;s also support the same set of
+     operations?</para>
+ 
+     <para>QUESTION: Some operations in <filename
+ 						 class="headerfile">src/Field/FieldOperators.h</filename> use both
+     &array; and &field;.  Do we list them here or in the &field;
+     section or both or somewhere else?</para>
+ 
+     <para>In the table below, &array; supplants the exact return types
+     because they are complicated and rarely need to be explicitly
+     written down.</para>
+ 
+     <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	   orient="port" pgwide="0">
+      <title>Operators on &array;</title>
+ 
+      <tgroup cols="2" align="left">
+       <thead>
+        <row>
+ 	<entry>Operator</entry>
+ 	<entry>Value</entry>
+        </row>
+       </thead>
+       <tbody>
+        <row>
+ 	<entry>
+           <funcsynopsis>
+ 	  <funcprototype>
+ 	   <funcdef>Array <function>acos</function></funcdef>
+ 	   <paramdef>const <parameter>Array&lt;Dim,T,EngineTag&gt;& a</parameter></paramdef>
+ 	  </funcprototype>
+ 	 </funcsynopsis>
+          </entry>
+ 	<entry><para>an array containing the element-wise inverse
+          cosine of the array <parameter>a</parameter></para></entry>
+        </row>
+        <row>
+ 	<entry>ADD ALL other operators appearing in <filename
+ 							      class="headerfile">src/Array/ArrayOperators.h</filename>,
+         <filename
+ 		  class="headerfile">src/Array/ArrayOperatorSpecializations.h</filename>,
+         <filename
+ 		  class="headerfile">src/Array/PoomaArrayOperators.h</filename>,
+         and <filename
+ 		      class="headerfile">src/Array/VectorArrayOperators.h</filename>.</entry>
+        </row>
+       </tbody>
+      </tgroup>
+     </table>
+ 
+     <para>FINISH: Write one or two examples or refer to ones
+     previously in the text.</para>
+    </section>
+ 
+ 
+    <section id="arrays_ref-reductions">
+     <title>Reducing All &array; Elements to One Value</title>
+ 
+     <para>These reduction functions repeatedly apply a binary
+     operation to all array elements to yield a value.  These functions
+     are similar to the Standard Template Library's
+     <function>accumulate</function> function.  For example,
+     <function>sum</function> repeatedly applies the binary plus
+     operator to all array elements, yielding the sum of all array
+     elements.</para>
+ 
+     <para>FINISH: What order of operation, if any, is
+     guaranteed?</para>
+ 
+     <para>FINISH: Add a table of the functions in <filename
+ 							    class="headerfile">src/Array/Reductions.h</filename>.</para>
+ 
+     <para>How does one use one's own binary function?  See <filename
+     class="headerfile">src/Engine/Reduction.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="arrays_ref-utilities">
+     <title>Utility Functions</title>
+ 
+     <section id="arrays_ref-utilities-compression">
+      <title>Compressed Data</title>
+ 
+      <para>Add a table containing
+      <function>elementsCompressed</function>,
+      <function>compressed</function>, <function>compress</function>,
+      and <function>uncompress</function>.</para>
+     </section>
+ 
+ 
+     <section id="arrays_ref-utilities-sizes">
+      <title>Centering Sizes and Number of Materials</title>
+ 
+      <para>ADD: a description of <function>numMaterials</function> and
+      <function>centeringSize</function> found in <filename
+      class="headerfile">src/Field/Field.h</filename>.  These functions
+      are meaningless for &array; but are provided for consistency with
+      &field;.</para>
+     </section>
+ 
+     <section id="arrays_ref-utilities-subfield">
+      <title>Obtaining Subfields</title>
+ 
+      <para>ADD: a description of <function>subField</function> found
+      in <filename class="headerfile">src/Field/Field.h</filename>.
+      This function, meaningless for &array;, is provided for
+      consistency with &field;.</para>
+     </section>
+    </section>
+ 
+ 
+    <section id="arrays_ref-tmp">
+     <title>TMP: What do we do with these &hellip;? Remove this
+     section.</title>
+ 
+     <para>QUESTION: Do we describe the &leaffunctor;s specialized for
+     &array;s in <filename
+     class="headerfile">src/Array/Array.h</filename> or in the &pete;
+     reference section?  What about the functions in <filename
+     class="headerfile">src/Array/CreateLeaf.h</filename>?</para>
+ 
+     <para>QUESTION: What is an <type>EngineFunctor</type>?  We
+     probably should describe it in an analogous way as for
+     &leaffunctor;s.</para>
+ 
+     <para>QUESTION: Where do we write about
+     <type>ExpressionTraits</type> for &array;s?</para>
+ 
+     <para>QUESTION: Do we describe the <type>ElementProperties</type>
+     specialization at this place or in its section?</para>
+ 
+     <para>QUESTION: Do we describe the <type>Patch</type>
+     specialization for &array;s (<filename
+     class="headerfile">src/Array/Array.h</filename>:1300) in this
+     place or in a section for patches?</para>
+    </section>
+   </chapter>
+ 
+ 
+   <chapter id="fields_ref">
+    <title>&field;s</title>
+ 
+    <para>An &array; is a set of values indexed by
+    coordinates, one value per coordinate.  It models the computer
+    science idea of an array.  Similarly, a &field; is a set of values
+    indexed by coordinate.  It models the mathematical and physical
+    idea of a field represented by a grid of rectangular cells, each
+    having at least one value.  A &field;'s functionality is a superset
+    of an &array;'s functionality because:
+     <itemizedlist>
+      <listitem>
+       <para>A &field; is distributed through space so one can compute
+       the distances between cells.</para>
+      </listitem>
+      <listitem>
+       <para>Each cell can hold multiple values.  For example, a
+       rectangular cell can have one value on each of its faces.</para>
+      </listitem>
+      <listitem>
+       <para>Multiple materials can share the same cell.  For example,
+       different values can be stored in the same cell for carbon,
+       oxygen, and nitrogen.</para>
+      </listitem>
+     </itemizedlist>
+    Also, &field;s' values can be related by relations.  Thus, if one
+    field's values change, a dependent field's values can be
+    automatically computed when needed.  FIXME: See also the unfinished
+    works chapter's entry concerning relations and arrays.</para>
+ 
+    <para>QUESTION: Should we add a picture comparing and contrasting
+    an array and a field?</para>
+ 
+    <para>QUESTION: How much structure can be copied from the &array;
+    chapter?</para>
+ 
+    <para>QUESTION: Where is <type>NewMeshTag</type>, defined in
+    <filename class="headerfile">src/Field/Field.h</filename>,
+    used?</para>
+ 
+    <para>QUESTION: Do we describe the &leaffunctor;s specialized for
+    &field;s in <filename
+    class="headerfile">src/Field/Field.h</filename> or in the &pete;
+    reference section?  Use the same decision for &array;s.</para>
+ 
+    <para>QUESTION: What do the structure and functions in <filename
+    class="headerfile">src/Field/Mesh/PositionFunctions.h</filename>
+    do?</para>
+ 
+ 
+    <section id="fields_ref-field">
+     <title>The &field; Container</title>
+ 
+     <para>ADD: table of template parameters and table of compile-time
+     types and values.</para>
+ 
+ 
+     <section id="fields_ref-field-constructors">
+      <title>Constructors and Destructors</title>
+ 
+      <para>ADD: this section similar to &array;s's constructor and
+      destructor section.</para>
+     </section>
+ 
+     <section id="fields_ref-field-initializers">
+      <title>Initializers</title>
+ 
+      <para>Add a table.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-access">
+      <title>Element Access</title>
+ 
+      <para>ADD: a table ala &array;.  Be sure to include
+      <methodname>all</methodname>.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-component">
+      <title>Component Access</title>
+ 
+      <para>ADD: a table ala &array;.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-subfields">
+      <title>Obtaining Subfields</title>
+ 
+      <para>ADD: discussion and a table listing ways to obtain
+       subfields.  Although the implementation may treat subfield views
+       and other field views similarly (?Is this true?), they are
+       conceptually different ideas so we present them
+       separately.</para>
+ 
+      <para>See <filename
+ 			 class="headerfile">src/Field/Field.h</filename>'s
+       <methodname>operator[]</methodname>,
+       <methodname>subField</methodname>, &hellip;,
+       <methodname>material</methodname>.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-relations">
+      <title>Supporting Relations</title>
+ 
+      <para>ADD: a table with the member functions including
+      <methodname>addRelation</methodname>,
+      <methodname>removeRelations</methodname>,
+      <methodname>applyRelations</methodname>, and
+      <methodname>setDirty</methodname>.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-accessors">
+      <title>Accessors</title>
+ 
+      <para>ADD: a table using lines like <filename
+      class="headerfile">src/Field/Field.h</filename>:1243&ndash;1333.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-utilities">
+      <title>Utility Methods</title>
+ 
+      <para>ADD: a table including
+       <methodname>makeOwnCopy</methodname>.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-field-implementation">
+      <title>Implementation Details</title>
+ 
+      <para>ADD: a table similar to &array;'s.</para>
+ 
+     </section>
+ 
+    </section>
+ 
+ 
+    <section id="engines_ref-views">
+     <title>Views of &field;s</title>
+ 
+     <para>Be sure to relate to &array; views.  Note only three
+     dimensions are supported.</para>
+ 
+     <para>Be sure to describe <statement>f[i]</statement>.  Does this
+     refer to a particular material or a particular value within a
+     cell?  I do not remember.  See <type>SubFieldView</type> in
+     <filename class="headerfile">src/Field/Field.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-assignment">
+     <title>&field; Assignments</title>
+ 
+     <para>ADD: Describe supported assignments, relating to &array;'s
+     assignments.</para>
+ 
+     <para>UNFINISHED: Add a table containing assignment operators
+     found on <filename
+ 		       class="headerfile">src/Field/Field.h</filename>:2097&ndash;2202
+     and 1512&ndash;1611.</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-print">
+     <title>Printing &field;s</title>
+ 
+     <para>QUESTION: How similar is this to printing &array;s?</para>
+ 
+     <para>&field;s support output to but not input from IO streams.
+     In particular, output to <type>ostream</type>s and file streams is
+     supported.</para>
+ 
+     <para>Add a table, using <filename
+ 				       class="headerfile">src/Field/Field.h</filename>, lines
+     1996&ndash;2009.  See the implementation in <filename
+ 							  class="headerfile">src/Field/PrintField.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-reductions">
+     <title>Combining &field; Elements</title>
+ 
+     <para>Like &array;s, &field;s support reduction of all elements to
+     one value.  Additionally, the latter supports computing a field's
+     values using field stencils.  QUESTION: How do I describe this
+     with a minimum of jargon?</para>
+ 
+     <para>ADD: something similar to &array; reductions.</para>
+ 
+     <para>FINISH: Add a table of the functions in <filename
+     class="headerfile">src/Field/FieldReductions.h</filename>.</para>
+ 
+     <para>FINISH: Add a table of the functions in <filename
+     class="headerfile">src/Field/DiffOps/FieldOffsetReductions.h</filename>.
+     QUESTION: Why is only <function>sum</function> defined?</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-expressions">
+     <title>Expressions Involving &field;s</title>
+ 
+     <para>Do something similar to &array;'s section.  See the
+     operations defined in <filename
+     class="headerfile">src/Field/FieldOperators.h</filename>,
+     <filename
+     class="headerfile">src/Field/FieldOperatorSpecializations.h</filename>,
+     <filename
+     class="headerfile">src/Field/PoomaFieldOperators.h</filename>, and
+     <filename
+     class="headerfile">src/Field/VectorFieldOperators.h</filename>.</para>
+ 
+     <para>Some operations involve both &array; and &field;
+     parameters.  Where do we list them?</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-stencils">
+     <title>&field; Stencils: Faster, Local Computations</title>
+ 
+     <para>ADD: a description of a stencil.  Why is it needed?  How
+     does a user use it?  How does a user know when to use one?  Add
+     documentation of the material from <filename
+     class="headerfile">src/Field/DiffOps/FieldStencil.h</filename>.</para>
+ 
+     <para>How is <type>FieldShiftEngine</type> used by &field;
+     stencils?  Should it be described here or in the &engine; section?
+     See the the code in <filename
+     class="headerfile">src/Field/DiffOps/FieldShiftEngine.h</filename>.</para>
+    </section>
+ 
+ 
+    <section>
+     <title>Cell Volumes, Face Areas, Edge Lengths, Normals</title>
+ 
+     <para>ADD: a description of these functions.  See <filename
+     class="headerfile">src/Field/Mesh/MeshFunctions.h</filename>.
+     These are initialized in, e.g., <filename
+     class="headerfile">src/Field/Mesh/UniformRectilinearMesh.h</filename>.
+     Note that these do not work for <type>NoMesh</type>.</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-div">
+     <title>Divergence Operators</title>
+ 
+     <para>ADD: a table having divergence operators, explaining the
+     current restrictions imposed by what is implemented.  See
+     <filename class="headerfile">src/Field/DiffOps/Div.h</filename>
+     and <filename
+     class="headerfile">src/Field/DiffOps/Div.UR.h</filename>.  What
+     restrictions does <quote>UR</quote> (mesh) connote?</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-utilities">
+     <title>Utility Functions</title>
+ 
+     <section id="fields_ref-utilities-compression">
+      <title>Compressed Data</title>
+ 
+      <para>Add a table containing
+      <function>elementsCompressed</function>,
+      <function>compressed</function>, <function>compress</function>,
+      and <function>uncompress</function>.</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-utilities-sizes">
+      <title>Centering Sizes and Number of Materials</title>
+ 
+      <para>ADD: a description of <function>numMaterials</function> and
+      <function>centeringSize</function> found in <filename
+      class="headerfile">src/Field/Field.h</filename>.</para>
+ 
+      <para>QUESTION: How do these relate to any method functions?</para>
+     </section>
+ 
+ 
+     <section id="fields_ref-utilities-subfield">
+      <title>Obtaining Subfields</title>
+ 
+      <para>ADD: a description of <function>subField</function> found
+      in <filename
+      class="headerfile">src/Field/Field.h</filename>.</para>
+     </section>
+ 
+    </section>
+ 
+ 
+    <section id="fields_ref-centerings">
+     <title>&field; Centerings</title>
+ 
+     <para>DO: Describe the purpose of a centering and its definition.
+     Describe the ability to obtain canonical centerings.  Explain how
+     to construct a unique centering.  See <filename
+     class="headerfile">src/Field/FieldDentering.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-fieldoffsets">
+     <title>Relative &field; Positions</title>
+ 
+     <para>Permit specifying field positions relative to a field
+     location.  Describe <type>FieldOffset</type> and
+     <type>FieldOffsetList</type>.  See <filename
+     class="headerfile">src/Field/FieldOffset.h</filename></para>
+    </section>
+ 
+ 
+    <section id="fields_ref-nearestneighbors">
+     <title>Computing Close-by Field Positions</title>
+ 
+     <para>Given a field location, return the set of field locations
+     that are closest using ?Manhattan? distance.  See <filename
+     class="headerfile">src/Field/NearestNeighbors.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="fields_ref-mesh">
+     <title>Mesh ???</title>
+ 
+     <para>Unlike &array;s, &field;s are distributed throughout space
+     so distances between values within the &field can be computed.  A
+     &field;'s mesh stores this spatial distribution.</para>
+ 
+     <para>QUESTION: What do we need to write about meshes?  What is
+     unimportant implementation and what should be described in this
+     reference section?</para>
+ 
+     <para>QUESTION: Where in here should emphasize vertex, not cell,
+     positions?  <quote>VERTEX</quote> appears repeatedly in <filename
+     class="headerfile">src/Field/Mesh/NoMesh.h</filename>.</para>
+ 
+     <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	   orient="port" pgwide="0">
+      <title>Mesh Types</title>
+ 
+      <tgroup cols="2" align="left">
+       <thead>
+        <row>
+ 	<entry>Mesh Type</entry>
+ 	<entry>Description</entry>
+        </row>
+       </thead>
+       <tbody>
+        <row>
+ 	<entry><type>NoMesh&lt;Dim&gt;</type></entry>
+ 	<entry><para>no physical spacing, causing a &field; to mimic
+ 	an &array; with multiple engines.</para></entry>
+        </row>
+        <row>
+ 	<entry><type>UniformRectilinearMesh&lt;Dim,T&gt;</type></entry>
+ 	<entry><para>physical spacing formed by the Cartesian product
+ 	of ????.</para></entry>
+        </row>
+       </tbody>
+      </tgroup>
+     </table>
+ 
+ 
+     <section id="fields_ref-mesh-accessors">
+      <title>Mesh Accessors</title>
+ 
+      <para>ADD: a table listing accessors, explaining the difference
+      between (physical and total) and (cell and vertex) domains.  See
+      <filename class="headerfile">src/Field/Mesh/NoMesh.h</filename>.
+      Also, include <function>spacings</function> and
+      <function>origin</function> in <filename
+      class="headerfile">src/Field/Mesh/UniformRectilinearMesh.h</filename>.
+      Note <type>NoMesh</type> does not provide the latter two.</para>
+     </section>
+ 
+    </section>
+ 
+ 
+    <section id="fields_ref-tmp">
+     <title>TMP: What do we do with these &hellip;? Remove this
+     section.</title>
+ 
+     <para>QUESTION: Do we describe the <type>Patch</type>
+     specialization for &field; at this place or in some common place?
+     Follow &array;'s lead.</para>
+ 
+     <para>QUESTION: Where do we describe <type>CreateLeaf</type> and
+     <type>MakeFieldReturn</type> in <filename
+     class="headerfile">src/Field/FieldCreateLeaf.h</filename> and
+     <filename
+     class="headerfile">src/Field/FieldMakeReturn.h</filename>.</para>
+ 
+     <para>QUESTION: What do we do with <type>FieldEnginePatch</type>
+     in <filename
+     class="headerfile">src/Field/FieldEngine/FieldEnginePatch.h</filename>.</para>
+    </section>
+   </chapter>
+ 
+ 
+   <chapter id="engines_ref">
+    <title>&engine;s</title>
+ 
+    <para>From a user's point of view, a container makes data available
+    for reading and writing.  In fact, the container's &engine; stores
+    the data or, if the data is computed, performs a computation to
+    yield the data.</para>
+ 
+    <para>FINISH: Introduce the various types of engines.  Add a table
+    with a short description of each engine type.</para>
+ 
+    <para>FINISH: First, we specify a generic &engine;'s interface.
+    Then, we present &engine; specializations.</para>
+    
+    <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	  orient="port" pgwide="0">
+     <title>Types of &engine;s</title>
+ 
+     <tgroup cols="3" align="left">
+      <thead>
+       <row>
+        <entry>Engine Type</entry>
+        <entry>Engine Tag</entry>
+        <entry>Description</entry>
+       </row>
+      </thead>
+      <tbody>
+       <row>
+        <entry><para>Brick</para></entry>
+        <entry><para><literal>Brick</literal></para></entry>
+        <entry><para>Explicitly store all elements in, e.g., a &cc;
+ 	array.</para></entry>
+       </row>
+       <row>
+        <entry><para>Compressible</para></entry>
+        <entry><para><literal>CompressibleBrick</literal></para></entry>
+        <entry><para>If all values are the same, use constant storage
+ 	for that single value.  Otherwise, explicitly store all
+ 	elements.</para></entry>
+       </row>
+       <row>
+        <entry><para>Constant</para></entry>
+        <entry><para><literal>ConstantFunction</literal></para></entry>
+        <entry><para>Returns the same constant value for all
+        indices.</para></entry>
+       </row>
+       <row>
+        <entry><para>Dynamic</para></entry>
+        <entry><para><literal>Dynamic</literal></para></entry>
+        <entry><para>Manages a contiguous, local, one-dimensional,
+        dynamically resizable block of data.</para></entry>
+       </row>
+       <row>
+        <entry><para>Component Forwarding</para></entry>
+        <entry><para><literal>CompFwd&lt;EngineTag,
+        Components&gt;</literal></para></entry>
+        <entry><para>Returns the specified components from
+        <literal>EngineTag</literal>'s engine.  Components are
+        <quote>pieces</quote> of multi-value elements such as vectors
+        and tensors.</para></entry>
+       </row>
+       <row>
+        <entry><para>Expression</para></entry>
+        <entry><para><literal>ExpressionTag&lt;Expr&gt;</literal></para></entry>
+        <entry><para>Returns the value of the specified &pete;
+        expression.</para></entry>
+       </row>
+       <row>
+        <entry><para>Index Function</para></entry>
+        <entry><para><literal>IndexFunction&lt;Functor&gt;</literal></para></entry>
+        <entry><para>Makes the function
+        <literal>Functor</literal>accepting indices mimic an
+        array.</para></entry>
+       </row>
+       <row>
+        <entry><para>MultiPatch</para></entry>
+        <entry><para><literal>MultiPatch&lt;LayoutTag,PatchTag&gt;</literal></para></entry>
+        <entry><para>Support distributed computation using several
+        processors (???contexts???).  <literal>LayoutTag</literal>
+        indicates how the entire array is distributed among the
+        processors.  Each processor uses a <literal>PatchTag</literal>
+        engine.</para></entry>
+       </row>
+       <row>
+        <entry><para>Remote</para></entry>
+        <entry><para><literal>Remote&lt;EngineTag&gt;</literal></para></entry>
+        <entry><para>unknown</para></entry>
+       </row>
+       <row>
+        <entry><para>Remote Dynamic</para></entry>
+        <entry><para><literal>Remote&lt;Dynamic&gt;</literal></para></entry>
+        <entry><para>unknown: specialization</para></entry>
+       </row>
+       <row>
+        <entry><para>Stencil</para></entry>
+        <entry><para><literal>StencilEngine&lt;Function,
+        Expression&gt;</literal></para></entry>
+        <entry><para>Returns values computed by applying the
+        user-specified function to sets of contiguous values in the
+        given engine or container.  Compare with user function
+        engines.</para></entry>
+       </row>
+       <row>
+        <entry><para>User Function</para></entry>
+        <entry><para><literal>UserFunctionEngine&lt;UserFunction,Expression&gt;</literal></para></entry>
+        <entry><para> Returns values computed by applying the
+        user-specified function to the given engine or container.
+        QUESTION: Is the following claim correct? For each returned
+        value, only one value from the engine or container is
+        used.</para></entry>
+       </row>
+      </tbody>
+     </tgroup>
+    </table>
+ 
+    <para>QUESTION: Where do we describe views?</para>
+ 
+    <para>QUESTION: What does <type>NewEngine</type> do?  Should it be
+     described when describing views?  Should it be omitted as an
+     implementation detail?</para>
+ 
+    <para>QUESTION: Where do we describe &engine; patches found in
+    <filename class="headerfile">src/Engine/EnginePatch.h</filename>?
+    All patch data in a separate chapter or engine-specific pieces in
+    this chapter?</para>
+ 
+    <para>QUESTION: What is <function>notifyEngineWrite</function>?
+    See also <filename
+ 		      class="headerfile">src/Engine/NotifyEngineWrite.h</filename>.</para>
+ 
+    <para>QUESTION: What aspect of MultiPatch uses <type>IsValid</type> in
+    <filename
+ 	     class="headerfile">src/Engine/IsValidLocation.h</filename>?</para>
+ 
+    <para>QUESTION: Who uses intersections?  Where should this be
+    described?  See <filename
+ 			     class="headerfile">src/Engine/Intersector.h</filename>, <filename
+ 											       class="headerfile">src/Engine/IntersectEngine.h</filename>, and
+    <filename
+ 	     class="headerfile">src/Engine/ViewEngine.h</filename>.</para>
+ 
+    <section id="engines_ref-compile_time_interface">
+     <title>&engine; Compile-Time Interface</title>
+ 
+     <para>ADD: a table of template parameters ala &array;.  ADD:
+     compile-time types and values.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-constructors">
+     <title>Constructors and Destructors</title>
+ 
+     <para>ADD: a table of constructors and destructors ala
+     &array;'s.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-access">
+     <title>Element Access</title>
+ 
+     <para>ADD: a table with <methodname>read</methodname> and
+     <methodname>operator()</methodname>.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-accessors">
+     <title>Accessors</title>
+ 
+     <para>ADD: a table of accessors.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-assignments">
+     <title>&engine; Assignments</title>
+ 
+     <para>similar to &array;'s assignments.  shallow copies.  ADD: a
+     table with one entry</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-utilities">
+     <title>Utility Methods</title>
+ 
+     <para>ADD: a table including
+     <methodname>makeOwnCopy</methodname>.</para>
+ 
+     <para>QUESTION: What are <methodname>dataObject</methodname>,
+     <methodname>isShared</methodname>, and related methods?</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-implementation">
+     <title>Implementation Details</title>
+ 
+     <para>ADD: this section.  Explain that
+     <varname>dataBlock_m</varname> and <varname>data_m</varname> point
+     to the same place.  The latter speeds access, but what is the
+     purpose of the former?</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-brick">
+     <title>Brick and BrickView Engines</title>
+ 
+     <para>ADD: description of what a brick means.  ADD: whatever
+     specializations the class has, e.g.,
+     <methodname>offset</methodname>.</para>
+ 
+     <para>QUESTION: What does <type>DoubleSliceHelper</type> do?</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-compressible">
+     <title>Compressible Brick and BrickView Engines</title>
+ 
+     <para>ADD this.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-dynamic">
+     <title>Dynamic and DynamicView Engines:</title>
+ 
+     <para>ADD this.  Manages a contiguous, local, resizable, 1D block
+     of data.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-components">
+     <title>Component Engines</title>
+ 
+     <para>I believe these implement array component-forwarding.  See
+     <filename
+ 	      class="headerfile">src/Engine/ForwardingEngine.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-expressions">
+     <title>Expression Engines</title>
+ 
+     <para>Should this be described in the &pete; section?  Unlikely.
+     See <filename
+ 		  class="headerfile">src/Engine/ExpressionEngine.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-enginefunctor">
+     <title>&engine; Functors</title>
+ 
+     <para>QUESTION: What is an <type>EngineFunctor</type>?  Should it
+     have its own section?  See <filename
+ 					 class="headerfile">src/Engine/EngineFunctor.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="engines_ref-fieldengines">
+     <title><type>FieldEngine</type>: A Hierarchy of &engine;s</title>
+ 
+     <para>A &field; consists of a hierarchy of materials and
+     centerings.  These are implemented using a hierarchy of engines.
+     See <filename
+ 		  class="headerfile">src/Field/FieldEngine/FieldEngine.h</filename>
+     and <filename
+ 		  class="headerfile">src/Field/FieldEngine/FieldEngine.ExprEngine.h</filename>.</para>
+    </section>
+   </chapter>
+ 
+ 
+   <chapter id="benchmarks_ref">
+    <title>&benchmark; Programs</title>
+ 
+    <para>Explain how to use &benchmark; programs, especially the
+    options.  Explain how to write a &benchmark; program.  See also
+    <filename class="headerfile">src/Utilities/Benchmark.h</filename>
+    and <filename
+    class="libraryfile">src/Utilities/Benchmark.cmpl.cpp</filename>.</para>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="layouts_ref">
+    <title>Layouts and Partitions: Distribute Computation Among
+    Contexts</title>
+ 
+    <para>What is the difference between <type>ReplicatedTag</type> and
+    <type>DistributedTag</type>?</para>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="pete_ref">
+    <title>&pete;: Evaluating Parallel Expressions</title>
+ 
+    <section>
+     <title>UNKNOWN</title>
+ 
+     <section id="pete_ref-unknown-leaf-tags">
+      <title>Leaf Tag Classes</title>
+ 
+      <para><type>NotifyPreReadTag</type> indicates a term is about to
+      be read.  Why is this needed?  Defined in <filename
+      class="headerfile">src/Utilities/NotifyPreRead.h</filename>.</para>
+     </section>
+    </section>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="views_ref">
+    <title>Views</title>
+ 
+    <para>QUESTION: Should this have its own chapter or be part of a
+    container chapter?</para>
+ 
+    <para>Describe <type>View0</type>, <type>View1</type>, &hellip;,
+    <type>View7</type> and <type>View1Implementation</type>.</para>
+ 
+    <para>QUESTION: What causes the need for <type>AltView0</type> and
+    <type>AltComponentView</type>?</para>
+ 
+    <para>Be sure to describe <type>ComponentView</type> in the same
+    place.  This is specialized for &array;s in <filename
+    class="headerfile">src/Array/Array.h</filename>:1323&ndash;1382.</para>
+ 
+    <section>
+     <title><type>ViewIndexer&lt;Dim,Dim2&gt;</type></title>
+ 
+     <para>Defined in <filename
+     class="headerfile">src/Utilities/ViewIndexer.h</filename>, this
+     type translates indices between a domain and a view of it.</para>
+    </section>
+   </chapter>
+ 
+   <chapter id="threads_ref">
+    <title>Threads</title>
+ 
+    <para>Perhaps include information in <filename
+    class="headerfile">src/Engine/DataObject.h</filename>.</para>
+ 
+    <para>&pooma; options include UNFINISHED</para>
+ 
+   </chapter>
+ 
+ 
+   <chapter id="utilities_ref">
+    <title>Utility Types</title>
+ 
+    <para>TMP: What is a good order?</para>
+ 
+    <section id="utilities_ref-options">
+     <title><type>Options</type>: Varying Run-Time Execution</title>
+ 
+     <para>Each &pooma; executable has a <type>Options</type> object,
+     created by <function>Pooma::initialize</function>, storing
+     run-time configurable values found in <varname>argv</varname>.
+     Default options are found in
+     <methodname>Options::usage</methodname>.</para>
+ 
+     <para>See <filename
+     class="headerfile">src/Utilities/Options.h</filename> and
+     <filename
+     class="libraryfile">src/Utilities/Options.cmpl.cpp</filename>.</para>
+ 
+     <para>Scatter the specific options to other parts of the
+     manual.</para>
+    </section>
+ 
+    <section id="utilities_ref-assertions">
+     <title>Check Correctness: <type>CTAssert</type>,
+     <type>PAssert</type>, <type>PInsist</type>,
+     <type>SameType</type></title>
+ 
+     <para>Assertions ensure program invariants are obeyed.
+     <type>CTAssert</type>, checked at compile time, incur no run-time
+     cost.  <type>PAssert</type> and <type>PInsist</type> are checked
+     to run-time, the latter producing an explanatory message if the
+     assertion fails.  Compiling with <envar>NOCTAssert</envar> and
+     <envar>NOPTAssert</envar> disable these checks.  Compiling with just
+     <envar>NOPTAssert</envar> disables only the run-time checks.</para>
+ 
+     <para><type>SameType</type> ensures, at compile-time, two types
+     are the same.</para>
+ 
+     <para>These are implemented in <filename
+     class="headerfile">src/Utilities/PAssert.h</filename> and
+     <filename
+     class="libraryfile">src/Utilities/PAssert.cmpl.cpp</filename>.</para>
+    </section>
+ 
+    <section id="utilities_ref-clock">
+     <title><type>Clock</type>: Measuring a Program's Execution Time</title>
+ 
+     <para>See <filename
+     class="headerfile">src/Utilities/Clock.h</filename>.</para>
+    </section>
+ 
+ 
+    <section id="utilities_ref-smart_pointers">
+     <title>Smart Pointers: <type>RefCountedPtr</type>,
+     <type>RefCountedBlockPtr</type>, and
+     <type>DataBlockPtr</type></title>
+ 
+     <para>See <filename
+     class="headerfile">src/Utilities/{RefCountedPtr,RefCountedBlockPtr,DataBlockPtr}.h</filename>.
+     <filename class="headerfile">src/Utilities/RefCounted.h</filename>
+     helps implement it.  <type>DataBlockPtr</type> uses
+     &smarts;.</para>
+    </section>
+ 
+    <section id="utilities_ref-inform">
+     <title><type>Inform</type>: Formatted Output for Multi-context
+     Execution</title>
+ 
+     <para>See <filename
+     class="headerfile">src/Utilities/Inform.h</filename> and <filename
+     class="libraryfile">src/Utilities/Inform.cmpl.cpp</filename>.</para>
+    </section>
+ 
+    <section id="utilities_ref-statistics">
+     <title><type>Statistics</type>: Report &pooma; Execution Statistics</title>
+ 
+     <para>Collect and print execution statistics.  Defined in
+     <filename
+     class="headerfile">src/Utilities/Statistics.h</filename>.</para>
+    </section>
+ 
+    <section id="utilities_ref-random_numbers">
+     <title>Random Numbers: <type>Unique</type></title>
+ 
+     <para>See <filename
+     class="headerfile">src/Utilities/Unique.h</filename>.</para>
+    </section>
+   </chapter>
+ 
+ 
+   <chapter id="implementation_types_ref">
+    <title>Types for Implementing &pooma;</title>
+ 
+    <para>TMP: What is a good order?</para>
+ 
+    <para>Describe types defined to implement &pooma; but that users do
+    not directly use.  This chapter has lower priority than other
+    chapters since users (hopefully) do not need to know about these
+    classes.</para>
+ 
+    <section id="implementation_types_ref-tester">
+     <title><type>Tester</type>: Check Implementation Correctness</title>
+ 
+     <para>&pooma; implementation test programs frequently consist of a
+     series of operations followed by correctness checks.  The
+     <type>Tester</type> object supports these tests, returning a
+     boolean whether all the correctness checks yield true.  Under
+     verbose output, messages are printed for each test.  See <filename
+     class="headerfile">src/Utilities/Tester.h</filename>.</para>
+    </section>
+ 
+    <section id="implementation_types_ref-elementproperties">
+     <title><type>ElementProperties&lt;T&gt;</type>: Properties a Type
+     Supports</title>
+ 
+     <para>This traits class permits optimizations in other templated
+     classes.  See <filename
+     class="headerfile">src/Utilities/ElementProperties.h</filename>.</para>
+ 
+    </section>
+ 
+    <section id="implementation_types_ref-typeinfo">
+     <title><type>TypeInfo&lt;T&gt;</type>: Print a String Describing
+     the Type</title>
+ 
+     <para>Print a string describing the type.  Defined in <filename
+     class="headerfile">src/Utilities/TypeInfo.h</filename>.  It is
+     specialized for other types in other files, e.g., <filename
+     class="headerfile">src/Engine/EngineTypeInfo.h</filename> and
+     <filename class="headerfile">src/Field/FieldTypeInfo.h</filename>.
+     Is this a compile-time version of RTTI?</para>
+    </section>
+ 
+    <section id="implementation_types_ref-looputils">
+     <title><type>LoopUtils</type>: Loop Computations at Compile Time</title>
+ 
+     <para>At compile time, <type>LoopUtils</type> supports copying
+     between arrays and computing the dot product of arrays.  See
+     <filename
+ 	      class="headerfile">src/Utilities/MetaProg.h</filename>.</para>
+    </section>
+ 
+    <section id="implementation_types_ref-modelelement">
+     <title><type>ModelElement&lt;T&gt;</type>: Wrap a Type</title>
+ 
+     <para>A wrapper class used to differentiate overloaded functions.
+     Defined in <filename
+     class="headerfile">src/Utilities/ModelElement.h</filename>.  Used
+     only by &array; and <type>DynamicArray</type>.</para>
+    </section>
+ 
+    <section id="implementation_types_ref-wrappedint">
+     <title><type>WrappedInt&lt;int&gt;</type>: Wrap a Number</title>
+ 
+     <para>A wrapper class used to differentiate overloaded functions
+     among different integers.  Defined in <filename
+     class="headerfile">src/Utilities/WrappedInt.h</filename>.  Is this
+     class deprecated?  Is it even necessary?</para>
+    </section>
+    
+    <section id="implementation_types_ref-empty_classes">
+     <title>Supporting Empty Classes</title>
+ 
+     <para>The <type>NoInit</type> tag class indicates certain
+     initializations should be skipped.  Defined in <filename
+     class="headerfile">src/Utilities/NoInit.h</filename>.</para>
+ 
+     <para>FIXME: Should be macro, not function.
+     <function>POOMA_PURIFY_CONSTRUCTORS</function> generates an empty
+     constructor, copy constructor, and destructor to avoid &purify;
+     warnings.  Defined in <filename
+     class="headerfile">src/Utilities/PurifyConstructors.h</filename>.</para>
+ 
+    </section>
+ 
+    <section id="implementation_types_ref-pool">
+     <title><type>Pooled&lt;T&gt;</type>: Fast Memory Allocation of
+     Small Blocks</title>
+ 
+     <para><type>Pooled&lt;T&gt;</type> speeds allocation and
+     deallocation of memory blocks for small objects with
+     type&nbsp;<type>T</type>.  Defined in <filename
+     class="headerfile">src/Utilities/Pooled.h</filename>, it is
+     implemented in <filename
+     class="headerfile">src/Utilities/Pool.h</filename> and <filename
+     class="libraryfile">src/Utilities/Pool.cmpl.cpp</filename>.
+     <filename class="headerfile">src/Utilities/StaticPool.h</filename>
+     no longer seems to be used.</para>
+    </section>
+ 
+    <section id="implementation_types_ref-uninitialized_vector">
+     <title><type>UninitializedVector&lt;T,Dim&gt;</type>: Create
+     Without Initializing</title>
+ 
+     <para>This class optimizes creation of an array of objects by
+     avoiding running the default constructors.  Later initialization
+     can occur, perhaps using a loop that can be unrolled.  Defined in
+     <filename
+     class="headerfile">src/Utilities/UninitializedVector.h</filename>,
+     this is used only by <type>DomainTraits</type>.</para>
+    </section>
+   </chapter>
+ 
+   <chapter id="implementation_algorithms_ref">
+    <title>Algorithms for Implementing &pooma;</title>
+ 
+    <para>In <filename>src/Utilities/algorithms.h</filename>,
+    <function>copy</function>, <function>delete_back</function>, and
+    <function>delete_shiftup</function> provide additional algorithms
+    using iterators.</para>
+   </chapter>
+ 
+ 
+   <chapter id="where-place-these_ref">
+    <title>TMP: Where do we describe these files?</title>
+ 
+    <itemizedlist>
+     <listitem>
+      <para><filename
+ 		     class="headerfile">src/Utilities/Conform.h</filename>: tag for
+      checking whether terms in expression have conforming
+      domains</para>
+     </listitem>
+ 
+     <listitem>
+      <para><filename
+ 		     class="headerfile">src/Utilities/DerefIterator.h</filename>:
+      <type>DerefIterator&lt;T&gt;</type> and
+      <type>ConstDerefIterator&lt;T&gt;</type> automatically
+      dereference themselves to maintain <literal>const</literal>
+      correctness.  <!-- FIXME: s/literal/keyword/ --></para>
+     </listitem>
+ 
+     <listitem>
+      <para><filename
+ 		     class="headerfile">src/Utilities/Observable.h</filename>,
+      <filename class="headerfile">src/Utilities/Observer.h</filename>,
+      and <filename
+ 		   class="headerfile">src/Utilities/ObserverEvent.h</filename>:
+      <type>Observable&lt;T&gt;</type>,
+      <type>SingleObserveable&lt;T&gt;</type>,
+      <type>Observer&lt;T&gt;</type>, and <type>ObserverEvent</type>
+      implement the observer pattern.  What is the observer pattern?
+      Where is this used in the code?</para>
+     </listitem>
+    </itemizedlist>
+ 
+   </chapter>
+ 
+  </part>
+ 
+ 
+  <appendix id="future_development">
+   <title>Future Development</title>
+ 
+   <section id="future_development-particles">
+    <title>Particles</title>
+ 
+    <para><filename
+ 		   class="libraryfile">docs/ParticlesDoc.txt</filename> has
+    out-of-date information.</para>
+ 
+    <para>See Section&nbsp;3.2.3 of
+    <filename>papers/pooma.ps</filename> for an out-of-date
+    description.</para>
+ 
+    <para><filename>papers/jvwr.ps</filename> concerns mainly
+    particles.  <filename>papers/8thSIAMPOOMAParticles.pdf</filename>,
+    by Julian Cummings and Bill Humphrey, concerns parallel particle
+    simulations.  <filename>papers/iscope98linac.pdf</filename>
+    describes a particle beam simulation using &pooma;; it mainly
+    concerns particles.</para>
+ 
+    <section>
+     <title>Particles</title>
+ 
+     <para>Do we want to include such a section?</para>
+ 
+     <para>Section&nbsp;3, "Sample Applications" of
+     <filename>papers/SiamOO98_paper.ps</filename> describes porting a
+     particle program written using High-Performance Fortran to
+     &pooma; and presumably why particles were added to &pooma;.  It
+     also describes <application>MC++</application>, a Monte Carlo
+     neutron transport code.</para>
+ 
+    </section>
+ 
+   </section>
+ 
+ 
+   <section id="future_development-composition_engine">
+    <title>Composition of &engine;s</title>
+ 
+    <para>The i,j-th element of the composition
+    <varname>a</varname>&compfn;<varname>b</varname> of two arrays
+    <varname>a</varname> and <varname>b</varname> equals a(b(i,j)).
+    The composition engine tagged <literal>IndirectionTag&lt;Array1,
+    Array2&gt;</literal>, defined in <filename
+ 					      class="headerfile">src/Engine/IndirectionEngine.h</filename> is
+    unfinished.</para>
+   </section>
+ 
+ 
+   <section id="future_development-container_consistency">
+    <title>Improving Consistency of Container Interfaces</title>
+ 
+    <section id="future_development-container_consistency-array_relations">
+     <title>Relations for &array;s</title>
+ 
+     <para>Do &array;s currently support relations?  If not, why not?
+    Should they be added?</para>
+    </section>
+ 
+    <section id="future_development-container_consistency-dimensions">
+     <title>Supporting the Same Number of Dimensions</title>
+ 
+     <para>&array; and &field; should support the same maximum number
+     of dimensions.  Currently, &array;s support seven dimensions and
+     &field;s support only three.  By definition, &dynamicarray;
+     supports only one dimension.</para>
+ 
+     <para>Relations for &array;s.</para>
+ 
+     <para>External guards for &array;s.</para>
+ 
+     <para>QUESTION: What is <quote>tiny</quote> about &matrix;?  Should
+     they be renamed?</para>
+    </section>
+ 
+   </section>
+ 
+ 
+   <section id="future_development-where">
+    <title><function>where</function> Proxies</title>
+ 
+    <para>QUESTION: Do we even discuss this broken
+    <quote>feature</quote>?  Where is it used?  Some related code is in
+    <filename>src/Array/Array.h</filename>:2511&ndash;2520.</para>
+   </section>
+ 
+ 
+   <section id="future_development-distributed_input">
+    <title>Easing Input for Distributed Programs</title>
+ 
+    <para>Currently, standard input to distributed programs is not
+    supported.  Instead input can be passed via command-line arguments,
+    which are replicated to each context.  &inform; streams support for
+    input could be added.  For context&nbsp;0, standard input could be
+    used.  Other contexts would use a <type>RemoteProxy</type> to
+    distribute the value to the other contests.  See <filename
+    class="headerfile">src/Engine/RemoteEngine.h</filename> for example
+    uses of <type>RemoteProxy</type>.</para>
+   </section>
+ 
+ 
+   <section id="future_development-cheetah_pooma_consistency">
+    <title>Improving Consistency Between &pooma; and &cheetah;</title>
+ 
+    <para>Improve the consistency between &cheetah;'s and &pooma;'s
+    configurations.  Currently, their defaults differ regarding
+    &cc; exceptions and static/shared libraries.</para>
+   </section>
+ 
+ 
+   <section id="future_development-long_term">
+    <title>Very Long Term Development Ideas</title>
+ 
+    <para>Describe how to write a new configuration file.</para>
+   </section>
+ 
+  </appendix>
+ 
+ 
+  <appendix id="installation">
+   <title>Obtaining and Installing &pooma;</title>
+ 
+   <para>ADD: Write this section, including extensive instructions
+   for Unix, MS Windows, and MacOS.  List the configuration options.
+   Be sure to describe configuring for parallel execution.</para>
+ 
+   <section id="installation-distributed_computing">
+    <title>Supporting Distributed Computation</title>
+ 
+    <para>To use multiple processors with &pooma; requires installing
+    the &cheetah; messaging library and an underlying messaging library
+    such as the Message Passing Interface (&mpi;) Communications
+    Library or the &mm; Shared Memory Library.  In this section, we
+    first describe how to install &mm;.  Read the section only if using
+    &mm;, not &mpi;.  Then we describe how to install &cheetah; and
+    configure &pooma; to use it.</para>
+ 
+    <section id="installation-distributed_computing-mm">
+     <title>Obtaining and Installing the &mm; Shared Memory Library</title>
+ 
+     <para>&cheetah;, and thus &pooma;, can use Ralf Engelschall's &mm;
+     Shared Memory Library to pass messages between processors.  For
+     example, the &author; uses this library on a two-processor
+     computer running &linux;.  The library, available at
+     http://www.engelschall.com/sw/mm/, is available for free and has
+     been successfully tested on a variety of Unix platforms.</para>
+ 
+     <para>We describe how to download and install the &mm; library.
+      <orderedlist spacing="compact">
+ 	<listitem>
+ 	 <para>Download the library from the &pooma; Download page
+          available off the &pooma; home page (&poomaHomePage;).</para>
+ 	</listitem>
+ 	<listitem>
+ 	 <para>Extract the source code using <command>tar xzvf
+          mm-1.1.3.tar.gz</command>.  Move into the resulting source
+          code directory <filename
+          class="directory">mm-1.1.3</filename>.</para>
+ 	</listitem>
+ 	<listitem>
+ 	 <para>Prepare to compile the source code by configuring it
+          using the <command>configure</command> command.  To change
+          the default installation directory <filename
+          class="directory">/usr/local</filename>, specify
+          <command>&dashdash;prefix=<replaceable>directory</replaceable></command>
+          option.  The other configuration options can be listed by
+          specifying the <command>&dashdash;help</command> option.  Since the
+          &author; prefers to keep all &pooma;-related code in his
+          <filename class="directory">pooma</filename>subdirectory, he
+          uses <command>./configure
+          &dashdash;prefix=${HOME}/pooma/mm-1.1.3</command>.</para>
+ 	</listitem>
+ 	<listitem>
+ 	 <para>Create the library by issuing the <command>make</command>
+      command.  This compiles the source code using a &c; compiler.  To
+      use a different compiler than the &mm; configuration chooses, set
+      the <envar>CC</envar> to the compiler before configuring.</para>
+       </listitem>
+     <listitem>
+      <para>Optionally test the library by issuing the <command>make
+      test</command> command.  If successful, the penultimate line
+      should be <computeroutput>OK - ALL TESTS SUCCESSFULLY
+      PASSED</computeroutput>.</para>
+     </listitem>
+     <listitem>
+      <para>Install the &mm; Library by issuing the <command>make
+      install</command> command.  This copies the library files to the
+      installation directory.  The <filename
+      class="directory">mm-1.1.3</filename> directory containing the
+      source code may now be removed.</para>
+     </listitem>
+    </orderedlist>
+    </para>
+    </section>
+ 
+    
+    <section id="installation-distributed_computing-cheetah">
+     <title>Obtaining and Installing the &cheetah; Messaging Library</title>
+ 
+     <para>The &cheetah; Library decouples communication from
+     synchronization.  Using asynchronous messaging rather than
+     synchronous messaging permits a message sender to operate without
+     the cooperation of the message recipient.  Thus, implementing
+     message sending is simpler and processing is more efficiently
+     overlapped with it.  Remote method invocation is also supported.
+     The library was developed at the Los Alamos National Laboratory's
+     Advanced Computing Laboratory.</para>
+ 
+     <para>&cheetah;'s messaging is implemented using an underlying
+     messaging library such as the Message Passing Interface (&mpi;)
+     Communications Library (FIXME: xref linkend="mpi99", <ulink
+     url="http://www-unix.mcs.anl.gov/mpi/"></ulink>) or the &mm;
+     Shared Memory Library.  &mpi; works on a wide variety of platforms
+     and has achieved widespread usage.  &mm; works under Unix on any
+     computer with shared memory.  Both libraries are available for
+     free.  The instructions below work for whichever library you
+     choose.</para>
+ 
+     <para>We describe how to download and install &cheetah;.
+      <orderedlist spacing="compact">
+       <listitem>
+        <para>Download the library from the &pooma; Download page
+        available off the &pooma; home page (&poomaHomePage;).</para>
+       </listitem>
+       <listitem>
+        <para>Extract the source code using <command>tar xzvf
+        cheetah-1.0.tgz</command>.  Move into the resulting source code
+        directory <filename
+        class="directory">cheetah-1.0</filename>.</para>
+       </listitem>
+       <listitem>
+        <para>Edit a configuration file corresponding to your operating
+        system and compiler.  These <filename
+        class="libraryfile">.conf</filename> files are located in the
+        <filename class="directory">config</filename> directory.  For
+        example, to use &gcc; with the &linux; operating system, use
+        <filename
+        class="libraryfile">config/LINUXGCC.conf</filename>.</para>
+ 
+        <para>The configuration file usually does not need
+        modification.  However, if you are using &mm;, ensure
+        <varname>shmem_default_dir</varname> specifies its location.
+        For example, the &author; modified the value to
+        <statement>"/home/oldham/pooma/mm-1.1.3"</statement>.</para>
+       </listitem>
+       <listitem>
+        <para>Prepare to compile the source code by configuring it
+        using the <command>configure</command> command.  Specify the
+        configuration file using the <command>&dashdash;arch</command> option.
+        Its argument should be the configuration file's name, omitting
+        its <filename class="libraryfile">.conf</filename> suffix.  For
+        example, <command>&dashdash;arch LINUXGCC</command>.  Some other
+        options include
+        <variablelist>
+        <varlistentry>
+ 	<term>&dashdash;help</term>
+ 	<listitem>
+ 	 <para>lists all the available options</para>
+ 	</listitem>
+        </varlistentry>
+        <varlistentry>
+ 	<term>&dashdash;shmem &dashdash;nompi</term>
+ 	<listitem>
+ 	 <para>indicates use of &mm;, not &mpi;</para>
+ 	</listitem>
+        </varlistentry>
+        <varlistentry>
+ 	<term>&dashdash;mpi &dashdash;noshmem</term>
+ 	<listitem>
+ 	 <para>indicates use of &mpi;, not &mm;</para>
+ 	</listitem>
+        </varlistentry>
+        <varlistentry>
+ 	<term>&dashdash;opt</term>
+ 	<listitem>
+ 	 <para>causes the compiler to produce optimized source code</para>
+ 	</listitem>
+        </varlistentry>
+        <varlistentry>
+ 	<term>&dashdash;noex</term>
+ 	<listitem>
+ 	 <para>prevents use of &cc; exceptions</para>
+ 	</listitem>
+        </varlistentry>
+        <varlistentry>
+ 	 <term>&dashdash;static</term>
+ 	 <listitem>
+ 	  <para>creates a static library, not a shared library</para>
+ 	 </listitem>
+        </varlistentry>
+        <varlistentry>
+ 	 <term>&dashdash;shared</term>
+ 	 <listitem>
+ 	  <para>creates a shared library, not a static library.  This
+ 	  is the default.</para>
+ 	 </listitem>
+        </varlistentry>
+        <varlistentry>
+ 	<term>&dashdash;prefix <replaceable>directory</replaceable></term>
+ 	<listitem>
+ 	 <para>specifies the installation directory where the
+ 	   library will be copied rather than the default.</para>
+ 	</listitem>
+        </varlistentry>
+       </variablelist>
+         For example, the &author; uses <command>./configure &dashdash;arch
+         LINUXGCC &dashdash;shmem &dashdash;nompi &dashdash;noex &dashdash;static &dashdash;prefix
+         ${HOME}/pooma/cheetah-1.0 &dashdash;opt</command>.  The
+         <command>&dashdash;arch LINUXGCC</command> indicates use of &gcc;
+         under a &linux; operating system.  The &mm; library is used,
+         but &cc; exceptions are not.  The latter choice matches
+         &pooma;'s default choice.  A static library, not a shared
+         library, is created.  This is also &pooma;'s default choice.
+         The library will be installed in the <filename
+         class="directory">${HOME}/pooma/cheetah-1.0</filename>.
+         Finally, the library code will be optimized, hopefully running
+         faster than unoptimized code.</para>
+       </listitem>
+       <listitem>
+        <para>Follow the directions printed by
+        <command>configure</command>: Change directories to the
+        <filename class="directory">lib</filename> subdirectory named
+        by the <command>&dashdash;arch</command> argument and then type
+        <command>make</command> to compile the source code and create
+        the library.</para>
+       </listitem>
+       <listitem>
+        <para>Optionally ensure the library works correctly by issuing
+        the <command>make tests</command> command.</para>
+       </listitem>
+       <listitem>
+        <para>Install the library by issuing the <command>make
+        install</command> command.  This copies the library files to
+        the installation directory.  The <filename
+        class="directory">cheetah-1.0</filename> directory containing
+        the source code may now be removed.</para>
+       </listitem>
+      </orderedlist>
+    </para>
+    </section>
+ 
+    <section id="installation-distributed_computing-pooma">
+     <title>Configuring &pooma; When Using &cheetah;</title>
+ 
+     <para>To use &pooma; with &cheetah;, one must tell &pooma; the
+     location of the &cheetah; library using the
+     <command>&dashdash;messaging</command> configuration option.  To do this,
+      <orderedlist spacing="compact">
+       <listitem>
+        <para>Set the &cheetah; directory environment variable
+         <envar>CHEETAHDIR</envar> to the directory containing the
+         installed &cheetah; library.  For
+         example, <command>declare -x
+         CHEETAHDIR=${HOME}/pooma/cheetah-1.0</command> specifies the
+         installation directory used in the previous section.</para>
+       </listitem>
+       <listitem>
+        <para>When configuring &pooma;, specify the
+        <command>&dashdash;messaging</command> option.  For example,
+        <command>./configure &dashdash;arch LINUXgcc &dashdash;opt
+        &dashdash;messaging</command> configures for &linux;, &gcc;, and an
+        optimized library using &cheetah;.</para>
+       </listitem>
+      </orderedlist>
+     </para>
+    </section>
+   </section>
+  </appendix>
+ 
+ 
+  <appendix id="compilation_errors">
+   <title>Dealing with Compilation Errors</title>
+ 
+   <para>Base this low-priority section on <filename
+   class="libraryfile">errors.html</filename>.  QUESTION: Where is
+   <filename class="libraryfile">errors.html</filename>?</para>
+ 
+  </appendix>
+ 
+ 
+  <appendix id="tmp_appendix">
+   <title>TMP: Notes to Myself</title>
+ 
+   <section id="tmp-miscellaneous">
+    <title>Miscellaneous</title>
+ 
+    <orderedlist>
+     <listitem>
+      <para>If there is time, present another example program, e.g., a
+      Jacobi solver.</para>
+     </listitem>
+ 
+     <listitem>
+      <para>If a reference manual for &pooma; implementors is written,
+      begin with a chapter <quote>Under the Hood: How &pooma;
+      Works</quote>, written from the point of view of &cc;
+      interpreter.  For &pete;, use the material in
+      <filename>papers/PETE_DDJ/ddj_article.html</filename>, which
+      gives example code and descriptions of how the code works, and
+      see material in <filename
+      class="libraryfile">background.html</filename>'s
+      <quote>Expression Templates.</quote></para>
+     </listitem>
+ 
+     <listitem>
+      <para>QUESTION: How do &pooma; parallel concepts compare with
+      Fortran&nbsp;D or high-performance Fortran FINISH CITE:
+      {koelbel94:_high_perfor_fortr_handb}?</para>
+     </listitem>
+ 
+     <listitem>
+      <para>QUESTION: How do I know when to use a type name versus just
+      the concept?  For example, when do I use <quote>array</quote>
+      versus &array;?</para>
+     </listitem>
+ 
+     <listitem>
+      <para>Krylov solvers are described in Section&nbsp;3.5.2 of
+      <filename>papers/pooma.ps</filename>.</para>
+     </listitem>
+ 
+     <listitem>
+      <para>Section&nbsp;5, "The Polygon Overlay Problem," describes
+      porting an <acronym>ANSI</acronym> &c; program to &pooma;.</para>
+     </listitem>
+ 
+     <listitem>
+      <para>A good example book: <emphasis>STL Tutorial and Reference
+      Guide: &cc; Programming with the Standard Template
+      Library</emphasis>, second edition, by David&nbsp;R. Musser,
+      Gillmer&nbsp;J. Derge, and Atul Sanai, ISBN 0-201-37923-6,
+      QA76.73.C153.M87 2001.</para>
+     </listitem>
+ 
+     <listitem>
+      <para>One STL reference book listed functions in margin notes,
+      easing finding material.  Do this.</para>
+     </listitem>
+ 
+     <listitem>
+      <para>QUESTION: Does Berna Massingill at Trinity University have
+      any interest ior access to any parallel computers?</para>
+     </listitem>
+    </orderedlist>
+ 
+   </section>
+ 
+ 
+   <section id="tmp-html_tutorials">
+    <title>Existing HTML Tutorials</title>
+ 
+    <para>All these tutorials are out-of-date, but the ideas and text
+    may still be relevant.</para>
+ 
+    <variablelist>
+     <varlistentry><term><filename
+ 				  class="libraryfile">index.html</filename></term>
+      <listitem><para>list of all tutorials.  No useful
+     material.</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+ 				  class="libraryfile">introduction.html</filename></term>
+      <listitem><para>data-parallel Laplace solver using Jacobi
+     iteration ala <command>Doof2d</command></para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+ 				  class="libraryfile">background.html</filename></term>
+      <listitem><para>short, indirect introduction to &pete;; parallel
+     execution model; &cc;; templates; &stl;; expression
+     templates</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+ 				  class="libraryfile">tut-01.html</filename></term>
+      <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+ 				  class="libraryfile">Layout.html</filename></term>
+      <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+ 				  class="libraryfile">parallelism.html</filename></term>
+      <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+ 				  class="libraryfile">self-test.html</filename></term>
+      <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">threading.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-03.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-04.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-05.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-06.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-07.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-08.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-09.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-10.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-11.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-12.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+     <varlistentry><term><filename
+     class="libraryfile">tut-13.html</filename></term>
+     <listitem><para>UNFINISHED</para></listitem>
+     </varlistentry>
+    </variablelist>
+ 
+   </section>
+ 
+  </appendix>
+ 
+ 
+  <!-- Bibliography -->
+ 
+  <bibliography id="bibliography">
+   <title>Bibliography</title>
+ 
+   <para>FIXME: How do I process these entries?</para>
+ 
+   <biblioentry>
+    <abbrev>mpi99</abbrev>
+    <authorgroup>
+     <author>
+      <firstname>William</firstname><surname>Gropp</surname>
+     </author>
+     <author>
+      <firstname>Ewing</firstname><surname>Lusk</surname>
+     </author>
+     <author>
+      <firstname>Anthony</firstname><surname>Skjellum</surname>
+     </author>
+    </authorgroup>
+    <copyright>
+     <year>1999</year>
+     <holder>Massachusetts Institute of Technology</holder>
+    </copyright>
+    <isbn>0-262-57132-3</isbn>
+    <publisher>
+     <publishername>The MIT Press</publishername>
+     <address>Cambridge, MA</address>
+    </publisher>
+    <title>Using MPI</title>
+    <subtitle>Portable Parallel Programming with the Message-Passing Interface</subtitle>
+    <edition>second edition</edition>
+   </biblioentry>
+  </bibliography>
+ 
+ 
+  &glossary-chapter; 
+ 
+  <!-- Index -->
+ 
+   &genindex.sgm;
+ 
+ </book>
Index: outline.xml
===================================================================
RCS file: outline.xml
diff -N outline.xml
*** /tmp/cvs1ybeCp	Tue Dec 11 13:31:11 2001
--- /dev/null	Fri Mar 23 21:37:44 2001
***************
*** 1,4287 ****
- <?xml version="1.0"?>
- 
- <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN" "file://usr/lib/sgml/docbookx.dtd" [
- 
- <!-- Text to Check, Revise, Replace -->
-   <!-- FIXME: Revise before releasing. -->
-   <!-- UPDATE: Check before publishing to see if any needs changing. -->
-   <!-- ADD: Write more material. -->
- 
- <!-- Index Entity Declarations -->
- <!ENTITY genindex.sgm SYSTEM "genindex.sgm">
- 
- <!-- General Entity Declarations -->
- 
- <!ENTITY book "book">
-   <!-- Produce a notation for the book/manual/report/WWW page.  -->
-   <!-- Modify this to the desired noun. -->
- <!ENTITY Book "Book">
-   <!-- Produce a capitalized version of &book;  -->
-   <!-- Modify this to the desired noun. -->
- <!ENTITY c "<application class='software'>C</application>">
-   <!-- Produce a notation for the C programming language.  -->
-   <!-- Modify this to the desired formatting. -->
- <!ENTITY cc "<application class='software'>C++</application>">
-   <!-- Produce a notation for the C++ programming language.  -->
-   <!-- Modify this to the desired formatting. -->
- <!ENTITY cheetah "<application class='software'>Cheetah</application>" >
-   <!-- Produce a notation for the Cheetah Library.  -->
- <!ENTITY doof2d "<command>Doof2d</command>" >
-   <!-- Produce a notation for the Doof2d program.  -->
- <!ENTITY make "<application class='software'>Make</application>">
-   <!-- Produce a notation for the GNU Make program.  -->
- <!ENTITY mm "<application class='software'>MM</application>">
-   <!-- Produce a notation for the MM Library.  -->
- <!ENTITY mpi "<application class='software'>MPI</application>">
-   <!-- Produce a notation for the MPI package.  -->
- <!ENTITY pdt "<application class='software'>PDToolkit</application>">
-   <!-- Produce a notation for the PDT software package.  -->
- <!ENTITY pete "<application class='software'>PETE</application>">
-   <!-- Produce a notation for the PETE library.  -->
- <!ENTITY pooma "<application class='software'>POOMA</application>">
-   <!-- Produce a notation for Pooma software.  -->
- <!ENTITY poomaToolkit "<application class='software'>POOMA Toolkit</application>">
-   <!-- Produce a notation for the Pooma toolkit.  -->
- <!ENTITY purify "<application class='software'>Purify</application>">
-   <!-- Produce a notation for the Purify library.  -->
- <!ENTITY smarts "<application class='software'>Smarts</application>">
-   <!-- Produce a notation for the Smarts software package.  -->
-   <!-- Modify this after capitalization is decided. -->
- <!ENTITY stl "<application class='software'>STL</application>">
-   <!-- Produce a notation for the C++ Standard Template Library software package.  -->
- <!ENTITY tau "<application class='software'>Tau</application>">
-   <!-- Produce a notation for the Tau software package.  -->
- 
- <!-- Type Entity Declarations -->
- 
- <!ENTITY array "<type>Array</type>">
-   <!-- The "Array" type. -->
- <!ENTITY benchmark "<type>Benchmark</type>">
-   <!-- The "Benchmark" type. -->
- <!ENTITY brick "<type>Brick</type>">
-   <!-- The "Brick" engine type. -->
- <!ENTITY compressiblebrick "<type>CompressibleBrick</type>">
-   <!-- The "CompressibleBrick" engine type. -->
- <!ENTITY distributedtag "<type>DistributedTag</type>">
-   <!-- The DistributedTag Layout type. -->
- <!ENTITY domain "<type>Domain</type>">
-   <!-- The "Domain" type. -->
- <!ENTITY double "<type>double</type>">
-   <!-- The C "double" type. -->
- <!ENTITY dynamicarray "<type>DynamicArray</type>">
-   <!-- The "DynamicArray" type. -->
- <!ENTITY engine "<type>Engine</type>">
-   <!-- The "Engine" type. -->
- <!ENTITY field "<type>Field</type>">
-   <!-- The "Field" type. -->
- <!ENTITY interval "<type>Interval</type>">
-   <!-- The "Interval" type. -->
- <!ENTITY layout "<type>Layout</type>">
-   <!-- The "Layout" type without template parameters. -->
- <!ENTITY leaffunctor "<type>LeafFunctor</type>">
-   <!-- The "LeafFunctor" type. -->
- <!ENTITY multipatch "<type>MultiPatch</type>">
-   <!-- The "MultiPatch" engine without template parameters. -->
- <!ENTITY replicatedtag "<type>ReplicatedTag</type>">
-   <!-- The ReplicatedTag Layout type. -->
- <!ENTITY stencil "<type>Stencil</type>">
-   <!-- The "Stencil" type. -->
- <!ENTITY vector "<type>Vector</type>">
-   <!-- The "Vector" type. -->
- 
- <!-- Mathematical Entity Declarations -->
- 
- <!ENTITY n "n">
-   <!-- the size of one dimension of an array -->
- 
- <!-- System and Operating System Entity Declarations -->
- <!ENTITY gcc "<application>g++</application>">
-   <!-- The GNU Compiler Collection C++ compiler. -->
- <!ENTITY kcc "<application>KCC</application>">
-   <!-- The KAI C++ compiler. -->
- <!ENTITY linux "<application>Linux</application>">
-   <!-- The Linux operating system. -->
- 
- <!-- &pooma; URLs and Files -->
- 
- <!ENTITY poomaDownloadPage '<ulink url="http://pooma.codesourcery.com/pooma/download">http://pooma.codesourcery.com/pooma/download</ulink>'>
-   <!-- The WWW page supporting downloading the &pooma; source code. -->
-   <!-- UPDATE this URL. -->
- <!ENTITY poomaHomePage '<ulink url="http://www.pooma.com/">http://www.pooma.com/</ulink>'>
-   <!-- The canonical Pooma home page. -->
-   <!-- UPDATE this filename. -->
- <!ENTITY poomaSource "pooma-2.3.0">
-   <!-- The Pooma source code directory. -->
- <!ENTITY poomaSourceFile "&poomaSource;.tgz">
-   <!-- The Pooma source code archive. -->
- <!ENTITY poomaExampleDirectory "examples/Manual">
-   <!-- The directory holding this manual's example codes. -->
- 
- <!-- Spelling and Formatting Decisions -->
- <!ENTITY author "author">
-   <!-- A word describing an author xor authors. -->
-   <!-- spelling: element-wise, not elementwise -->
-   <!-- phrase: function object, not functor -->
-   <!-- spelling: multidimensional, not multi-dimensional -->
-   <!-- spelling: multiprocessor, not multi-processor -->
-   <!-- spelling: nonzero, not non-zero -->
- 
- <!-- External File Entities -->
- <!ENTITY doof2d-c-element SYSTEM "./programs/Doof2d-C-element-annotated.cpp">
-   <!-- hand-coded Doof2d implementation -->
- <!ENTITY doof2d-array-element SYSTEM "./programs/Doof2d-Array-element-annotated.cpp">
-   <!-- Array element-wise Doof2d implementation -->
- <!ENTITY doof2d-array-parallel SYSTEM "./programs/Doof2d-Array-parallel-annotated.cpp">
-   <!-- Array data-parallel Doof2d implementation -->
- <!ENTITY doof2d-array-stencil SYSTEM "./programs/Doof2d-Array-stencil-annotated.cpp">
-   <!-- Array stencil Doof2d implementation -->
- <!ENTITY doof2d-array-distributed SYSTEM "./programs/Doof2d-Array-distributed-annotated.cpp">
-   <!-- distributed Array stencil Doof2d implementation -->
- ]>
- 
- <book>
-  <bookinfo>
-   <title>&pooma;</title>
-   <subtitle>A &cc; Toolkit for High-Performance Parallel Scientific Computing</subtitle>
-   <author><firstname>Jeffrey</firstname><othername
-   role='mi'>D.</othername><surname>Oldham</surname>
-    <affiliation>
-     <orgname>CodeSourcery, LLC</orgname>
-    </affiliation>
-   </author>
-   <copyright><year>2001</year><holder>CodeSourcery, LLC (<ulink url="http://www.codesourcery.com"></ulink>)</holder></copyright>
-   <contractsponsor>Los Alamos National Laboratory<ulink url="http://www.lanl.gov"></ulink></contractsponsor>
-   <legalnotice>
-    <!-- FIXME: What is the correct legal notice? -->
-    <para>All rights reserved.  This document may not be redistributed in any form without the express permission of the author.</para>
-   </legalnotice>
-   <revhistory>
-    <revision>
-     <revnumber>0.01</revnumber>
-     <date>2001 Nov 26</date>
-     <authorinitials>jdo</authorinitials>
-     <revremark>first draft</revremark>
-    </revision>
-   </revhistory>
-  </bookinfo>
- 
-  <!-- FINISH: May we have a short table of contents followed by a -->
-  <!-- complete table of contents? -->
- 
-  <preface id="preface">
-   <title>Preface</title>
- 
-   <para>FINISH: Describe the target audience for &pooma; programs and
-   for this manual: &cc; programmers writing scientific code, possibly
-   parallel execution.</para>
- 
-   <para>Assume familiarity with &cc; template programming and the
-   standard template library.  FIXME: Remove this index
-   entry.<indexterm id="oldham"><primary>Oldham,
-   Jeffrey&nbsp;D.</primary></indexterm></para>
- 
-   <section id="preface-notation">
-    <title>Notation</title>
- 
-    <para>UNFINISHED</para>
-   </section>
- 
- 
-   <section id="preface-reading_book:">
-    <title>How to Read This &Book;</title>
- 
-    <para>FINISH: Write this section in a style similar to Lamport's
-    LaTeX section 1.2.  FINISH: Fix the book title and the section
-    number.</para>
-   </section>
- 
- 
-   <section id="preface-downloading">
-    <title>Obtaining &pooma; and Sample Programs</title>
- 
-    <para>Available for free from what WWW site?  Include what portions
-    of <filename class="libraryfile">LICENSE</filename>?  Be sure to
-    include CVS instructions as well.</para>
- 
-    <para>Which additional packages are necessary and when?</para>
- 
-   </section>
- 
- 
-   <section id="preface-using_modifying">
-    <title>Using and Modifying &pooma;</title>
- 
-    <para>&pooma; is available under open source license.  It can be
-    used and modified by anyone, anywhere.  Can it be sold?  Include
-    <filename class="libraryfile">LICENSE</filename>.</para>
- 
-    <para>QUESTION: How do developers contribute code?</para>
- 
-   </section>
- 
-  </preface>
- 
- 
-  <part id="programming">
-   <title>Programming with &pooma;</title>
- 
-   <chapter id="introduction">
-    <title>Introduction</title>
- 
-    <para>QUESTION: Add a partintro to the part above?</para>
- 
-    <para>&pooma; abbreviates <quote>Parallel Object-Oriented Methods
-    and Application</quote>.</para>
- 
-    <para>This document is an introduction to &pooma; v2.1, a &cc;
-    toolkit for high-performance scientific computation.  &pooma;
-    runs efficiently on single-processor desktop machines,
-    shared-memory multiprocessors, and parallel supercomputers
-    containing dozens or hundreds of processors. What's more, by making
-    extensive use of the advanced features of the ANSI/ISO &cc;
-    standard&mdash;particularly templates&mdash;&pooma; presents a
-    compact, easy-to-read interface to its users.</para>
- 
-    <para>From Section&nbsp; of
-    <filename>papers/iscope98.pdf</filename>:</para>
- 
-    <para>Scientific software developers have struggled with the need
-    to express mathematical abstractions in an elegant and maintainable
-    way without sacrificing performance.  The &pooma; (Parallel
-    Object-Oriented Methods and Applications) framework, written in
-    <acronym>ANSI</acronym>/<acronym>ISO</acronym> &cc;, has
-    demonstrated both high expressiveness and high performance for
-    large-scale scientific applications on platforms ranging from
-    workstations to massively parallel supercomputers.  &pooma; provides
-    high-level abstractions for multidimensional arrays, physical
-    meshes, mathematical fields, and sets of particles.  &pooma; also
-    exploits techniques such as expression templates to optimize serial
-    performance while encapsulating the details of parallel
-    communication and supporting block-based data compression.
-    Consequently, scientists can quickly assemble parallel simulation
-    codes by focusing directly on the physical abstractions relevant to
-    the system under study and not the technical difficulties of
-    parallel communication and machine-specific optimization.</para>
- 
-    <para>ADD: diagram of science and &pooma;.  See the diagram that
-    Mark and I wrote.</para>
- 
- 
-    <section id="introduction-pooma_evolution">
-     <title>Evolution of &pooma;</title>
- 
-     <para>QUESTION: Is this interesting?  Even if it is, it should be
-     short.</para>
- 
-     <para>The file <filename>papers/SCPaper-95.html</filename>
-     describes ?&pooma;1? and its abstraction layers.</para>
- 
-     <para>The "Introduction" of
-     <filename>papers/Siam0098.ps</filename> describes the DoE's
-     funding motivation for &pooma;: Accelerated Strategic Computing
-     Initiative (ASCI) and Science-based Stockpile Stewardship (SBSS),
-     pp. 1&ndash;2.</para>
- 
-     <para>See list of developers on p.&nbsp;1 of
-     <filename>papers/pooma.ps</filename>.</para>
- 
-     <para>See list of developers on p.&nbsp;1 of
-     <filename>papers/pooma.ps</filename>.  See history and motivation
-     on p.&nbsp;3 of <filename>papers/pooma.ps</filename>.</para>
- 
-     <para>Use <filename class="libraryfile">README</filename> for
-     information.</para>
- 
-     <blockquote>
-      <attribution><filename
- 			    class="libraryfile">introduction.html</filename></attribution>
- 
-      <para>&pooma; was designed and implemented by scientists working
-      at the Los Alamos National Laboratory's Advanced Computing
-      Laboratory. Between them, these scientists have written and tuned
-      large applications on almost every commercial and experimental
-      supercomputer built in the last two decades. As the technology
-      used in those machines migrates down into departmental computing
-      servers and desktop multiprocessors, &pooma; is a vehicle for its
-      designers' experience to migrate as well. In particular,
-      &pooma;'s authors understand how to get good performance out of
-      modern architectures, with their many processors and multi-level
-      memory hierarchies, and how to handle the subtly complex problems
-      that arise in real-world applications.</para>
-     </blockquote>
- 
-    </section>
- 
-   </chapter>
- 
- 
-   <chapter id="tutorial">
-    <title>A Tutorial Introduction</title>
- 
-    <para>UPDATE: In the following paragraph, fix the cross-reference
-    to the actual section.</para>
- 
-    <para>&pooma; provides different containers and processor
-    configurations and supports different implementation styles, as
-    described in <xref linkend="introduction"></xref>.  In this
-    chapter, we present several different implementations of the
-    &doof2d; two-dimensional diffusion simulation program:
-     <itemizedlist spacing="compact">
-      <listitem>
-       <para>a C-style implementation omitting any use of &pooma;
-       computing each array element individually,</para>
-      </listitem>
-      <listitem>
-       <para>a &pooma; &array; implementation computing each array
-       element individually,</para>
-      </listitem>
-      <listitem>
-       <para>a &pooma; &array; implementation using data-parallel
-       statements,</para>
-      </listitem>
-      <listitem>
-       <para>a &pooma; &array; implementation using stencils, which
-       support local computations,</para>
-      </listitem>
-      <listitem>
-       <para>a stencil-based &pooma; &array; implementation supporting
-       computation on multiple processors</para>
-      </listitem>
-      <listitem>
-       <para>a &pooma; &field; implementation using data-parallel
-       statements, and</para>
-      </listitem>
-      <listitem>
-       <para>a data-parallel &pooma; &field; implementation for
-       multi-processor execution.</para>
-      </listitem>
-     </itemizedlist>
-    </para>
-    <para>These illustrate the &array;, &field;, &engine;, layout,
-    mesh, and domain data types.  They also illustrate various
-    immediate computation styles (element-wise accesses, data-parallel
-    expressions, and stencil computation) and various processor
-    configurations (one sequential processor and multiple
-    processors).</para>
- 
-    <figure float="1" id="tutorial-doof2d_averagings">
-     <title>&doof2d; Averagings</title>
-     <mediaobject>
-      <imageobject>
-       <imagedata fileref="figures/doof2d.201" format="EPS" align="center"></imagedata>
-      </imageobject>
-      <textobject>
-       <phrase>The Initial Configuration</phrase>
-      </textobject>
-     </mediaobject>
-     <mediaobject>
-      <imageobject>
-       <imagedata fileref="figures/doof2d.202" format="EPS"></imagedata>
-      </imageobject>
-      <textobject>
-       <phrase>After the First Averaging</phrase>
-      </textobject>
-     </mediaobject>
-     <mediaobject>
-      <imageobject>
-       <imagedata fileref="figures/doof2d.203" format="EPS"></imagedata>
-      </imageobject>
-      <textobject>
-       <phrase>After the Second Averaging</phrase>
-      </textobject>
-     </mediaobject>
-    </figure>
- 
-    <para>The &doof2d; diffusion program starts with a two-dimensional
-    grid of values.  To model an initial density, all grid values are
-    zero except for one nonzero value in the center.  Each averaging,
-    each grid element, except the outermost ones, updates its value by
-    averaging its value and its eight neighbors.  To avoid overwriting
-    grid values before all their uses occur, we use two arrays, reading
-    the first and writing the second and then reversing their roles
-    within each iteration.</para>
- 
-    <para>Figure <xref linkend="tutorial-doof2d_averagings"></xref>
-    illustrates the averagings.  Initially, only the center element has
-    nonzero value.  To form the first averaging, each element's new
-    value equals the average of its and its neighbors' previous values.
-    Thus, the initial nonzero value spreads to a three-by-three grid.
-    The averaging continues, spreading to a five-by-five grid of
-    nonzero values.  Values in outermost grid cells are always
-    zero.</para>
- 
-    <para>Before presenting various implementations of %doof2d;, we
-    explain how to install the &poomaToolkit;.</para>
- 
-    <para>REMOVE: &doof2d; algorithm and code is illustrated in
-    Section&nbsp;4.1 of
-    <filename>pooma-publications/pooma.ps</filename>.  It includes a
-    figure illustrating parallel communication of data.</para>
- 
-    <section id="tutorial-installation">
-     <title>Installing &pooma;</title>
- 
-     <para>ADD: How does one install &pooma; using Windows or Mac?</para>
- 
-     <para>UPDATE: Make a more recent &pooma; source code file
-     available on &poomaDownloadPage;.  For example,
-     <quote>LINUXgcc.conf</quote> is not available.</para>
- 
-     <para>In this section, we describe how to obtain, build, and
-     install the &poomaToolkit;.  We focus on installing under the
-     Unix operating system.  Instructions for installing on computers
-     running Microsoft Windows or MacOS, as well as more extensive
-     instructions for Unix, appear in <xref
-     linkend="installation"></xref>.</para>
- 
-     <para>Obtain the &pooma; source code <filename
-     path="http://www.codesourcery.com/pooma/downloads_folder/">&poomaSourceFile;</filename>
-     from the &pooma; download page (&poomaDownloadPage;) available off
-     the &pooma; home page (&poomaHomePage;).  The <quote>tgz</quote>
-     indicates this is a compressed tar archive file.  To extract the
-     source files, use <command>tar xzvf &poomaSourceFile;</command>.
-     Move into the source code directory <filename
-     class="directory">&poomaSource;</filename> directory; e.g.,
-     <command>cd &poomaSource;</command>.</para>
- 
-     <para>Configuring the source code prepares the necessary paths for
-     compilation.  First, determine a configuration file in
-     corresponding to your operating system and compiler in the
-     <filename class="directory">config/arch/</filename> directory.
-     For example, <filename
-     class="libraryfile">LINUXgcc.conf</filename> supports compiling
-     under a &linux; operating system with &gcc; and <filename
-     class="libraryfile">SGI64KCC.conf</filename> supports compiling
-     under a 64-bit <application>SGI</application> Unix operating
-     system with &kcc;.  Then, configure the source code:
-     <command>./configure --arch LINUXgcc --opt --suite
-     LINUXgcc-opt</command>.  The architecture argument to the
-     <command>--arch</command> option is the name of the corresponding
-     configuration file, omitting its <filename
-     class="libraryfile">.conf</filename> suffix.  The
-     <command>--opt</command> indicates the &poomaToolkit; will
-     contain optimized source code, which makes the code run more
-     quickly but may impede debugging.  Alternatively, the
-     <command>--debug</command> option supports debugging.  The
-     <glossterm linkend="glossary-suite_name">suite name</glossterm>
-     can be any arbitrary string.  We chose
-     <command>LINUXgcc-opt</command> to remind us of the architecture
-     and optimization choice.  <filename
-     class="libraryfile">configure</filename> creates subdirectories
-     named by the suite name <quote>LINUXgcc-opt</quote> for use when
-     compiling the source files.  Comments at the beginning of
-     <filename
-     class="libraryfile">lib/<replaceable>suiteName</replaceable>/PoomaConfiguration.h</filename>
-     record the configuration arguments.</para>
- 
-     <para>To compile the source code, set the
-     <envar>POOMASUITE</envar> environment variable to the suite name
-     and then type <command>make</command>.  To set the environment
-     variable for the <application>bash</application> shell use
-     <command>export
-     POOMASUITE=<replaceable>suiteName</replaceable></command>,
-     substituting the suite name's
-     <replaceable>suiteName</replaceable>.  For the
-     <application>csh</application> shell, use <command>setenv
-     POOMASUITE LINUXgcc-opt</command>.  Issuing the
-     <command>make</command> command compiles the &pooma; source code
-     files to create the &pooma; library.  The &pooma; makefiles assume
-     the <trademark>GNU</trademark> &make; so substitute the proper
-     command if necessary.  The &pooma; library can be found in, e.g.,
-     <filename
-     class="libraryfile">lib/LINUXgcc-opt/libpooma-gcc.a</filename>.</para>
-    </section>
- 
-    <section id="tutorial-hand_coded">
-     <title>Hand-Coded Implementation</title>
- 
-     <para>Before implementing &doof2d; using the &poomaToolkit;, we
-     present a hand-coded implementation of &doof2d;.  See <xref
-     linkend="tutorial-hand_coded-doof2d"></xref>.  After querying the
-     user for the number of averagings, the arrays' memory is
-     allocated.  Since the arrays' size is not known at compile time,
-     the arrays are accesses via pointers to allocated dynamic memory.
-     This memory is deallocated at the program's end to avoid memory
-     leaks.  The arrays are initialized with initial conditions.  For
-     the <varname>b</varname> array, all values except the central ones
-     have nonzero values.  Only the outermost values of the
-     <varname>a</varname> array need be initialized to zero, but we
-     instead initialize them all using the loop used by
-     <varname>b</varname>.</para>
- 
-     <para>The simulation's kernel consists of triply nested loops.
-     The outermost loop controls the number of iterations.  The inner
-     nested loops iterate through the arrays' elements, excepting the
-     outermost elements; note the loop indices range from 1 to n-2
-     while the array indices range from 0 to n-1.  Each
-     <varname>a</varname> value is assigned the average of its
-     corresponding value in <varname>b</varname> and the latter's
-     neighbors.  Values in the two-dimensional grids are accessed using
-     two sets of brackets, e.g., <statement>a[i][j]</statement>.  After
-     assigning values to <varname>a</varname>, a second averaging reads
-     values in <varname>a</varname>, writing values in
-     <varname>b</varname>.</para>
- 
-     <para>After the kernel finishes, the final central value is
-     printed.  If the desired number of averagings is even, the value
-     in <varname>b</varname> is printed; otherwise, the value in
-     <varname>a</varname> is used.  Finally, the dynamically-allocated
-     memory must be freed to avoid memory leaks.</para>
- 
-     <example id="tutorial-hand_coded-doof2d">
-      <title>Hand-Coded Implementation of &doof2d;</title>
-      &doof2d-c-element;
-      <calloutlist>
-       <callout arearefs="tutorial-hand_coded-doof2d-nuaveragings">
-        <para>The user specifies the desired number of averagings.</para>
-       </callout>
-       <callout arearefs="tutorial-hand_coded-doof2d-array_storage">
-        <para>These variables point to the two-dimensional,
-        dynamically-allocated grids so we use a pointer to a pointer to
-        a &double;.</para>
-       </callout>
-       <callout arearefs="tutorial-hand_coded-doof2d-grid_size">
-        <para>The user enters the desired grid size.  The grid will be
-        a square with <varname>n</varname> by <varname
-        lang="c++">n</varname> grid cells.</para>
-       </callout>
-       <callout arearefs="tutorial-hand_coded-doof2d-allocation">
-        <para>Memory for the arrays is allocated.  By default, the
-        array indices are zero-based.</para>
-       </callout>
-       <callout arearefs="tutorial-hand_coded-doof2d-initialization">
-        <para>Initially, all grid values are zero except for the one
-        nonzero value at the center of the second array.  Array
-        positions are indicated using two brackets, e.g.,
-        <statement>a[i][j]</statement>.  A better implementation might
-        initialize only the outermost values of the
-        <varname>a</varname> array.</para>
-       </callout>
-       <callout arearefs="tutorial-hand_coded-doof2d-constants">
-        <para>These constants indicate the number of iterations, and
-        the average weighting.</para>
-       </callout>
-       <callout arearefs="tutorial-hand_coded-doof2d-first_write">
-        <para>Each <varname>a</varname> value, except an outermost one,
-        is assigned the average of its analogous <varname>b</varname>
-        value and that value's neighbors.  Note the loop indices ensure
-        the outermost values are not changed.  The
-        <varname>weight</varname>'s value ensures the computation is an
-        average.</para>
-       </callout>
-       <callout arearefs="tutorial-hand_coded-doof2d-second_write">
-        <para>The second averaging computes <varname>b</varname>'s
-        values using values stored in <varname>a</varname>.</para>
-       </callout>
-       <callout arearefs="tutorial-hand_coded-doof2d-answer">
-        <para>After the averagings finish, the central value is printed.</para>
-       </callout>
-       <callout arearefs="tutorial-hand_coded-doof2d-deallocation">
-        <para>The dynamically-allocated memory must be deallocated to
-        avoid memory leaks.</para>
-       </callout>
-      </calloutlist>
-     </example>
- 
-     <para>To compile the executable, change directories to the &pooma;
-     <filename
-     class="directory">&poomaExampleDirectory;/Doof2d</filename>
-     directory.  Ensure the <envar>POOMASUITE</envar> environment
-     variable specifies the desired suite name
-     <replaceable>suiteName</replaceable>, as we did when compiling
-     &pooma; in the previous section <xref
-     linkend="tutorial-installation"></xref>.  Issuing the
-     <command>make Doof2d-C-element</command> command creates the
-     executable
-     <command><replaceable>suiteName</replaceable>/Doof2d-C-element</command>.</para>
- 
-     <para>When running the executable, specify the desired a
-     nonnegative number of averagings and the nonnegative number of
-     grid cells along any dimension.  The resulting grid has the same
-     number of cells along each dimension.  After the executable
-     finishes, the resulting value of the central element is
-     printed.</para>
-    </section>
- 
- 
-    <section id="tutorial-array_elementwise">
-     <title>Element-wise &array; Implementation</title>
- 
-     <para>The simplest way to use the &poomaToolkit; is to
-     use the &pooma; &array; class instead of &c; arrays.  &array;s
-     automatically handle memory allocation and deallocation, support a
-     wider variety of assignments, and can be used in expressions.
-     <xref linkend="tutorial-array_elementwise-doof2d"></xref>
-     implements &doof2d; using &array;s and element-wise accesses.
-     Since the same algorithm is used as <xref
-     linkend="tutorial-hand_coded-doof2d"></xref>, we will concentrate
-     on the differences.</para>
- 
-     <example id="tutorial-array_elementwise-doof2d">
-      <title>Element-wise &array; Implementation of &doof2d;</title>
-      &doof2d-array-element;
-      <calloutlist>
-       <callout arearefs="tutorial-array_elementwise-doof2d-header">
-        <para>To use &pooma; &array;s, the <filename
-        class="headerfile">Pooma/Arrays.h</filename> must be included.</para>
-       </callout>
-       <callout arearefs="tutorial-array_elementwise-doof2d-pooma_initialize">
-        <para>The &poomaToolkit; structures must be constructed before
-        their use.</para>
-       </callout>
-       <callout arearefs="tutorial-array_elementwise-doof2d-domain">
-        <para>Before creating an &array;, its domain must be specified.
-        The <varname>N</varname> interval represents the
-        one-dimensional integral set {0, 1, 2, &hellip;, n-1}.  An
-        <type>Interval&lt;2&gt;</type> object represents the entire
-        two-dimensional index domain.</para>
-       </callout>
-       <callout arearefs="tutorial-array_elementwise-doof2d-array_creation">
-        <para>An &array;'s template parameters indicate its dimension,
-        its value type, and how the values will be stored or computed.
-        The &brick; &engine; type indicates values will be directly
-        stored.  It is responsible for allocating and deallocating
-        storage so <statement>new</statement> and
-        <statement>delete</statement> statements are not necessary.
-        The <varname>vertDomain</varname> specifies the array index
-        domain.</para>
-       </callout>
-       <callout arearefs="tutorial-array_elementwise-doof2d-initialization">
-        <para>The first statement initializes all &array; values to the
-        same scalar value.  This is possible because each &array;
-        <quote>knows</quote> its domain.  The second statement
-        illustrates &array; element access.  Indices, separated by
-        commas, are surrounded by parentheses rather than surrounded by
-        square brackets (<statement>[]</statement>).</para>
-       </callout>
-       <callout arearefs="tutorial-array_elementwise-doof2d-first_write">
-        <para>&array; element access uses parentheses, rather than
-        square brackets</para>
-       </callout>
-       <callout arearefs="tutorial-array_elementwise-doof2d-blockAndEvaluate">
-        <para>&pooma; may reorder computation of statements.  Calling
-        <function>Pooma::blockAndEvaluate</function> ensures all
-        computation finishes before accessing a particular array
-        element.</para>
-       </callout>
-       <callout arearefs="tutorial-array_elementwise-doof2d-deallocation">
-        <para>Since &array;s are first-class objects, they
-        automatically deallocate any memory they require, eliminating
-        memory leaks.</para>
-       </callout>
-       <callout arearefs="tutorial-array_elementwise-doof2d-pooma_finish">
-        <para>The &poomaToolkit; structures must be destructed after
-        their use.</para>
-       </callout>
-      </calloutlist>
-     </example>
- 
-     <para>We describe the use of &array; and the &poomaToolkit; in
-     <xref linkend="tutorial-array_elementwise-doof2d"></xref>.
-     &array;s, declared in the <filename
-     class="headerfile">Pooma/Arrays.h</filename>, are first-class
-     objects.  They <quote>know</quote> their index domain, can be used
-     in expressions, can be assigned scalar and array values, and
-     handle their own memory allocation and deallocation.</para>
- 
-     <para>The creation of the <varname>a</varname> and
-     <varname>b</varname> &array;s requires an object specifying their
-     index domains.  Since these are two-dimensional arrays, their
-     index domains are also two dimensional.  The two-dimensional
-     <type>Interval&lt;2&gt;</type> object is the Cartesian product of
-     two one-dimensional <type>Interval&lt;1&gt;</type> objects, each
-     specifying the integral set {0, 1, 2, &hellip;, n-1}.</para>
- 
-     <para>An &array;'s template parameters indicate its dimension, the
-     type of its values, and how the values are stored.  Both
-     <varname>a</varname> and <varname>b</varname> are two-dimension
-     arrays storing &double;s so their <varname>dimension</varname>
-     is&nbsp;2 and its element type is &double;.  An &engine; stores an
-     &array;'s values.  For example, a &brick; &engine; explicitly
-     stores all values.  A &compressiblebrick; &engine; also explicitly
-     stores values if more than value is present, but, if all values
-     are the same, storage for just that value is required.  Since an
-     engine can store its values any way it desires, it might instead
-     compute its values using a function or compute the values stored
-     in separate engines.  In practice, most explicitly specified
-     &engine;s are either &brick; or &compressiblebrick;.</para>
- 
-     <para>&array;s support both element-wise access and scalar
-     assignment.  Element-wise access uses parentheses, not square
-     brackets.  For example, <statement>b(n/2,n/2)</statement>
-     specifies the central element.  The scalar assignment <statement>b
-     = 0.0</statement> assigns the same 0.0 value to all array
-     elements.  This is possible because the array knows the extent of
-     its domain.</para>
- 
-     <para>After the kernel finishes, the central value is printed out.
-     Just prior to this &array; access, a call to
-     <statement>Pooma::blockAndEvaluate()</statement> ensures all
-     computation has finished.  &pooma; may reorder computation or
-     distribute them among various processors.  Before reading an
-     individual &array; value, <function>blockAndEvaluate</function>
-     ensures the value has the correct value.  Calling this function is
-     necessary only when accessing individual array elements because
-     &pooma; cannot determine when to call the function itself. For
-     example, before printing an array, &pooma; will call
-     <function>blockAndEvaluate</function> itself.</para>
- 
-     <para>Any program using the &poomaToolkit; must initialize the
-     toolkit's data structures using
-     <statement>Pooma::initialize(argc,argv)</statement>.  This
-     extracts &pooma;-specific command-line options from the
-     command-line arguments in <varname>argv</varname> and initializes
-     the inter-processor communication and other data structures.  When
-     finished, <statement>Pooma::finalize()</statement> ensures all
-     computation has finished and the communication and other data
-     structures are destructed.</para>
-    </section>
- 
- 
-    <section id="tutorial-data_parallel">
-     <title>Data-Parallel &array; Implementation</title>
- 
-     <para>&pooma; supports data-parallel &array; accesses.  Many
-     algorithms are more easily expressed using data-parallel
-     expressions.  Also, the &poomaToolkit; might be able to reorder
-     the data-parallel computations to be more efficient or distribute
-     them among various processors.  In this section, we concentrate
-     the differences between the data-parallel implementation of
-     &doof2d; listed in <xref
-     linkend="tutorial-array_parallel-doof2d"></xref> and the
-     element-wise implementation listed in the previous section <xref
-     linkend="tutorial-array_elementwise"></xref>.</para>
- 
-     <example id="tutorial-array_parallel-doof2d">
-      <title>Data-Parallel &array; Implementation of &doof2d;</title>
-      &doof2d-array-parallel;
-      <calloutlist>
-       <callout arearefs="tutorial-array_parallel-doof2d-innerdomain">
-        <para>These variables specify one-dimensional domains {1, 2,
-        &hellip;, n-2}.  Their Cartesian product specifies the domain
-        of the array values that are modified.</para>
-       </callout>
-       <callout arearefs="tutorial-array_parallel-doof2d-first_write">
-        <para>Data-parallel expressions replace nested loops and array
-        element accesses.  For example, <statement>a(I,J)</statement>
-        represents the subset of the <varname>a</varname> array having
-        a domain equal to the Cartesian product of <varname>I</varname>
-        and <varname>J</varname>.  Intervals can shifted by an additive
-        or multiplicative constant.</para>
-       </callout>
-      </calloutlist>
-     </example>
- 
-     <para>Data-parallel expressions apply domain objects to containers
-     to indicate a set of parallel expressions.  For example, in the
-     program listed above, <statement>a(I,J)</statement> specifies all
-     of <varname>a</varname> array excepting the outermost elements.
-     The array's <varname>vertDomain</varname> domain consists of the
-     Cartesian product of {0, 1, 2, &hellip;, n-1} and itself, while
-     <varname>I</varname> and <varname>J</varname> each specify {1, 2,
-     &hellip;, n-2}.  Thus, <statement>a(I,J)</statement> is the subset
-     with a domain of the Cartesian product of {1, 2, &hellip;, n-2}
-     and itself.  It is called a <firstterm>view</firstterm> of an
-     array.  It is itself an array, with a domain and supporting
-     element access, but its storage is the same as
-     <varname>a</varname>'s.  Changing a value in
-     <statement>a(I,J)</statement> also changes the same value in
-     <varname>a</varname>.  Changing a value in the latter also changes
-     the former if the value is not one of <varname>a</varname>'s
-     outermost elements.  The expression
-     <statement>b(I+1,J+1)</statement> indicates the subset of
-     <varname>b</varname> with a domain consisting of the Cartesian
-     product of {2, 3, &hellip;, n-1}, i.e., the same domain as
-     <statement>a(I,J)</statement> but shifted up one unit and to the
-     right one unit.  Only an &interval;'s value, not its name, is
-     important.  Thus, all uses of <varname>J</varname> in this program
-     could be replaced by <varname>I</varname> without changing the
-     semantics.</para>
- 
-     <figure float="1" id="tutorial-array_parallel-doof2d-adding_arrays">
-      <title>Adding &array;s</title>
-      <mediaobject>
-       <imageobject>
-        <imagedata fileref="figures/doof2d.210" format="EPS" align="center"></imagedata>
-       </imageobject>
-       <textobject>
-        <phrase>Adding two arrays with different domains.</phrase>
-       </textobject>
-       <caption>
-        <para>When adding arrays, values in corresponding positions are
-        added even if they have different indices, indicated by the
-        small numbers adjacent to the arrays.</para>
-       </caption>
-      </mediaobject>
-     </figure>
- 
-     <para>The statement assigning to <statement>a(I,J)</statement>
-     illustrates that &array;s may participate in expressions.  Each
-     addend is a view of an array, which is itself an array.  Each view
-     has the same domain size so their sum can be formed by
-     corresponding elements of each array.  For example, the lower,
-     left element of the result equals the sum of the lower, left
-     elements of the addend arrays.  For the computation, indices are
-     ignored; only the relative positions within each domain are used.
-     <xref
-     linkend="tutorial-array_parallel-doof2d-adding_arrays"></xref>
-     illustrates adding two arrays with different domain indices.  The
-     indices are indicated by the small numbers to the left and the
-     bottom of the arrays.  Even though 9 and 3 have different indices
-     (1,1) and (2,0), they are added to each other because they have
-     the same relative positions within the addends.</para>
-    </section>
- 
- 
-    <section id="tutorial-stencil">
-     <title>Stencil &array; Implementation</title>
- 
-     <para>Many computations are local, computing a &array;'s value by
-     using close-by &array; values.  Encapsulating this computation in
-     a stencil can yield faster code because the compiler can determine
-     all accesses come from the same array.  Each stencil consists of a
-     function object and an indication of the stencil's extent.</para>
- 
-     <example id="tutorial-array_stencil-doof2d">
-      <title>Stencil &array; Implementation of &doof2d;</title>
-      &doof2d-array-stencil;
-      <calloutlist>
-       <callout arearefs="tutorial-array_stencil-doof2d-stencil">
-        <para>A stencil is a function object implementing a local
-        operation on an &array;.</para>
-       </callout>
-       <callout
-        arearefs="tutorial-array_stencil-doof2d-stencil_operator">
-        <para>&pooma; applies this function call
-        <function>operator()</function> to the interior domain of an
-        &array;.  Although not strictly necessary, the function's
-        template parameter <varname>C</varname> permits using this
-        stencil with &array;s and other containers.  The
-        <function>read</function> &array; member function supports only
-        reading values, not writing values, thus possibly permitting
-        faster access.</para>
-       </callout>
-       <callout arearefs="tutorial-array_stencil-doof2d-stencil_extent">
-        <para>These two functions indicate the stencil's size.  For
-        each dimension, the stencil extends one cell to the left of (or
-        below) its center and also one call to the right (or above) its
-        center.</para>
-       </callout>
-       <callout
-        arearefs="tutorial-array_stencil-doof2d-stencil_creation">
-        <para>Create the stencil.</para>
-       </callout>
-       <callout arearefs="tutorial-array_stencil-doof2d-first_write">
-        <para>Applying <varname>stencil</varname> to the
-        <varname>b</varname> array and a subset
-        <varname>interiorDomain</varname> of its domain yields an
-        array, which is assigned to a subset of <varname>a</varname>.
-        The stencil's function object is applied to each position in
-        the specified subset of <varname>b</varname>.</para>
-       </callout>
-      </calloutlist>
-     </example>
- 
-     <para>Before we describe how to create a stencil, we describe how
-     to apply a stencil to an array, yielding values.  To compute the
-     value associated with index position (1,3), the stencil's center
-     is placed at (1,3).  The stencil's
-     <function>upperExtent</function> and
-     <function>lowerExtent</function> functions indicate which &array;
-     values the stencil's function will use.  See <xref
-     linkend="tutorial-array_stencil-doof2d-apply_stencil"></xref>.
-     Applying the stencil's function call
-     <function>operator()</function> yields the computed value.  To
-     compute multiple &array; values, apply a stencil to the array and
-     a domain object: <statement>stencil(b,
-     interiorDomain)</statement>.  This applies the stencil to each
-     position in the domain.  The user must ensure that applying the
-     stencil does not access nonexistent &array; values.</para>
- 
-     <figure float="1" id="tutorial-array_stencil-doof2d-apply_stencil">
-      <title>Applying a Stencil to an &array;</title>
-      <mediaobject>
-       <imageobject>
-        <imagedata fileref="figures/doof2d.211" format="EPS" align="center"></imagedata>
-       </imageobject>
-       <textobject>
-        <phrase>Apply a stencil to position (1,3) of an array.</phrase>
-       </textobject>
-       <caption>
-        <para>To compute the value associated with index position (1,3)
-        of an array, place the stencil's center, indicated with dashed
-        lines, at the position.  The computation involves the array
-        values covered by the array and delineated by
-        <function>upperExtent</function> and
-        <function>lowerExtent</function>.</para>
-       </caption>
-      </mediaobject>
-     </figure>
- 
-     <para>To create a stencil object, apply the &stencil; type to a
-     function object class.  For example,
-     <statement>Stencil&lt;DoofNinePt&gt; stencil</statement> declares
-     the <varname>stencil</varname> object.  The function object class
-     must define a function call <function>operator()</function> with a
-     container parameter and index parameters.  The number of index
-     parameters, indicating the stencil's center, must equal the
-     container's dimension.  For example, <type>DoofNinePt</type>
-     defines <methodname>operator()(const C& c, int i, int
-     j)</methodname>.  We templated the container type
-     <varname>C</varname> although this is not strictly necessary.  The
-     two index parameters <varname>i</varname> and <varname>j</varname>
-     ensure the stencil works with two-dimensional containers.  The
-     <methodname>lowerExtent</methodname> indicates how far to the left
-     (or below) the stencil extends beyond its center.  Its parameter
-     indicates a particular dimension.  Index parameters
-     <varname>i</varname> and <varname>j</varname> are in dimension 0
-     and&nbsp;1.  <methodname>upperExtent</methodname> serves an
-     analogous purpose.  The &poomaToolkit; uses these functions when
-     distribution computation among various processors, but it does not
-     use these functions to ensure nonexistent &array; values are not
-     accessed.  Caveat stencil user!</para>
-    </section>
- 
- 
-    <section id="tutorial-distributed">
-     <title>Distributed &array; Implementation</title>
- 
-     <para>A &pooma; program can execute on one or multiple processors.
-     To convert a program designed for uniprocessor execution to a
-     program designed for multiprocessor execution, the programmer need
-     only specify how each container's domain should be split into
-     <quote>patches</quote>.  The &poomaToolkit; automatically
-     distributes the data among the available processors and handles
-     any required communication between processors.</para>
- 
-     <example id="tutorial-array_distributed-doof2d">
-      <title>Distributed Stencil &array; Implementation of &doof2d;</title>
-      &doof2d-array-distributed;
-      <calloutlist>
-       <callout arearefs="tutorial-array_distributed-doof2d-nuprocessors">
-        <para>The number of processors executing a &pooma; program can
-        be specified at run-time.</para>
-       </callout>
-       <callout arearefs="tutorial-array_distributed-doof2d-layout">
-        <para>The <type>UniformGridPartition</type> declaration
-        specifies how an array's domain will be partition, of split,
-        into patches.  Guard layers are an optimization that can reduce
-        data communication between patches.  The
-        <type>UniformGridLayout</type> declaration applies the
-        partition to the given domain, distributing the resulting
-        patches among various processors.</para>
-       </callout>
-       <callout arearefs="tutorial-array_distributed-doof2d-remote">
-        <para>The <type>MultiPatch</type> &engine; distributes requests
-        for &array; values to the associated patch.  Since a patch may
-        associated with a different processor, its
-        <quote>remote</quote> engine has type
-        <type>Remote&lt;Brick&gt;</type>.  &pooma; automatically
-        distributes the patches among available memories and
-        processors.</para>
-       </callout>
-       <callout
-        arearefs="tutorial-array_distributed-doof2d-first_write">
-        <para>The stencil computation, whether for one processor or
-        multiple processors, is the same.</para>
-       </callout>
-      </calloutlist>
-     </example>
- 
-     <para>Supporting distributed computation requires only minor code
-     changes.  These changes specify how each container's domain is
-     distributed among the available processors.  The rest of the
-     program, including all the computations, remains the same.  When
-     running, the &pooma; executable interacts with the run-time
-     library to determine which processors are available, distributes
-     the containers' domains, and automatically handles all necessary
-     interprocessor communication.  The same executable runs on one or
-     many processors.  Thus, the programmer can write one program,
-     debugging it on a uniprocessor computer and running it on a
-     supercomputer.</para>
- 
-     <figure float="1" id="tutorial-array_distributed-doof2d-distributed_model">
-      <title>The &pooma; Distributed Computation Model</title>
-      <mediaobject>
-       <imageobject>
-        <imagedata fileref="figures/distributed.101" format="EPS" align="center"></imagedata>
-       </imageobject>
-       <textobject>
-        <phrase>the &pooma; distributed computation model.</phrase>
-       </textobject>
-       <caption>
-        <para>The &pooma; distributed computation model combines
-        partitioning containers' domains and the computer configuration
-        to create a layout.</para>
-       </caption>
-      </mediaobject>
-     </figure>
- 
-     <para>&pooma;'s distributed computing model separates container
-     domain concepts from computer configuration concepts.  See <xref
-     linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>.
-     The program indicates how each container's domain will be
-     partitioned.  This process is represented in the upper left corner
-     of the figure.  A user-specified partition specifies how to split
-     the domain into pieces.  For example, the illustrated partition
-     splits the domain into three equal-sized pieces along the
-     x-dimension and two equal-sized pieces along the y-dimension.
-     Thus, the domain is split into <firstterm>patches</firstterm>.
-     The partition also specifies external and internal guard layers.
-     A <firstterm>guard layer</firstterm> is a domain surrounding a
-     patch.  A patch's computation only reads but does not write these
-     values.  An <firstterm>external guard layer</firstterm>
-     conceptually surrounds the entire container domain with boundary
-     values whose presence permits all domain computations to be
-     performed the same way even for values along the domain's edge.
-     An <firstterm>internal guard layer</firstterm> duplicates values
-     from adjacent patches so communication need not occur during a
-     patch's computation.  The use of guard layers is an optimization;
-     using external guard layers eases programming and using internal
-     guard layers reduces communication between processor.  Their use
-     is not required.</para>
- 
-     <para>The computer configuration of shared memory and processors
-     is determined by the run-time system.  See the upper right portion
-     of <xref
-     linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>.
-     A <firstterm>context</firstterm> is a collection of shared memory
-     and processors that can execute a program or a portion of a
-     program.  For example, a two-processor desktop computer might have
-     memory accessible to both processors so it is a context.  A
-     supercomputer consisting of desktop computers networked together
-     might have as many contexts as computers.  The run-time system,
-     e.g., the Message Passing Interface (&mpi;) Communications Library
-     (FIXME: xref linkend="mpi99", <ulink
-     url="http://www-unix.mcs.anl.gov/mpi/"></ulink>) or the &mm;
-     Shared Memory Library (<ulink
-     url="http://www.engelschall.com/sw/mm/"></ulink>), communicates
-     the available contexts to the executable.  &pooma; must be
-     configured for the particular run-time system.  See <xref
-     linkend="installation-distributed_computing"></xref>.</para>
- 
-     <para>A <firstterm>layout</firstterm> combines patches with
-     contexts so the program can be executed.  If &distributedtag; is
-     specified, the patches are distributed among the available
-     contexts.  If &replicatedtag; is specified, each set of patches is
-     replicated among each context.  Regardless, the containers'
-     domains are now distributed among the contexts so the program can
-     run.  When a patch needs data from another patch, the &pooma;
-     toolkit sends messages to the desired patch uses a message-passing
-     library.  All such communication is automatically performed by the
-     toolkit with no need for programmer or user input.</para>
- 
-     <para>FIXME: The two previous paragraphs demonstrate confusion
-     between <quote>run-time system</quote> and <quote>message-passing
-     library</quote>.</para>
- 
-     <para>Incorporating &pooma;'s distributed computation model into a
-     program requires writing very few lines of code.  <xref
-     linkend="tutorial-array_distributed-doof2d"></xref> illustrates
-     this.  The <varname>partition</varname> declaration creates a
-     <type>UniformGridPartition</type> splitting each dimension of a
-     container's domain into equally-sized
-     <varname>nuProcessors</varname> pieces.  The first
-     <type>GuardLayers</type> argument specifies each patch will have
-     copy of adjacent patches' outermost values.  This may speed
-     computation because a patch need not synchronize its computation
-     with other patches' processors.  Since each value's computation
-     requires knowing its surrounding neighbors, the internal guard
-     layer is one layer deep.  The second <type>GuardLayers</type>
-     argument specifies no external guard layer.  External guard layers
-     simplify computing values along the edges of domains.  Since the
-     program already uses only the interior domain for computation, we
-     do not use this feature.</para>
- 
-     <para>The <varname>layout</varname> declaration creates a
-     <type>UniformGridLayout</type> layout.  As <xref
-     linkend="tutorial-array_distributed-doof2d"></xref> illustrates,
-     it needs to know a container's domain, a partition, the computer's
-     contexts, and a &distributedtag; or &replicatedtag;.  These
-     comprise <varname>layout</varname>'s three parameters; the
-     contexts are implicitly supplied by the run-time system.</para>
- 
-     <para>To create a distributed &array;, it should be created using
-     a &layout; object and have a &multipatch; engine.  Prior
-     implementations designed for uniprocessors constructed the
-     container using a &domain; object.  A distributed implementation
-     uses a &layout; object, which conceptually specifies a &domain;
-     object and its distribution throughout the computer.  A
-     &multipatch; engine supports computations using multiple patches.
-     The <type>UniformTag</type> indicates the patches all have the
-     same size.  Since patches may reside on different contexts, the
-     second template parameter is <type>Remote</type>.  Its
-     <type>Brick</type> template parameter specifies the engine for a
-     particular patch on a particular context.  Most distributed
-     programs use <type>MultiPatch&lt;UniformTag, Remote&lt;Brick&gt;
-     &gt;</type> or <type>MultiPatch&lt;UniformTag,
-     Remote&lt;CompressibleBrick&gt; &gt;</type> engines.</para>
- 
-     <para>The computations for a distributed implementation are
-     exactly the same as for a sequential implementation.  The &pooma;
-     Toolkit and a message-passing library automatically perform all
-     computation.</para>
- 
-     <para>The command to run the programs is dependent on the run-time
-     system.  To use &mpi; with the Irix&nbsp;6.5 operating system, one
-     can use the <command>mpirun</command> command.  For example,
-     <statement>mpirun -np 9 Doof2d-Array-distributed -mpi
-     --num-patches 3</statement> invokes the &mpi; run-time system with
-     nine processors.  The <statement>-mpi</statement> argument tells
-     the &pooma; executable <command>Doof2d-Array-distributed</command>
-     to use the &mpi; Library.  
- 
- HERE
- 
-     The command <statement>Doof2d-Array-distributed -shmem -np 2
-     --num-patches 2</statement> 
- 
-     To run <command>Doof2d-Array-distributed</command> with the &mm;
-     Shared Memory Library, use 
- 
- HERE</para>
- 
- <!-- HERE -->
- 
-     <para>COMMENT: See <filename
-     class="libraryfile">background.html</filename> for a partial
-     explanation.  A <firstterm>context</firstterm> is a distinct
-     region of memory in some computer.  Execution thread is associated
-     with each context.  One or more different processors can be
-     associated with the same context.</para>
- 
-     <para>QUESTION: How do &pooma; parallel concepts compare with
-     Fortran&nbsp;D or high-performance Fortran FINISH CITE:
-     {koelbel94:_high_perfor_fortr_handb}?</para>
- 
-     <para>QUESTION: What does Cheetah do for us?  Must configure with
-     --messaging and Cheetah library must be available.  When running
-     Doof2d benchmark, use --num-patches N.  On LinuxKCC, use
-     '--num-patches p --run-impls 14 --sim-params N 0 1'.  Runtime
-     system must also provide some support.  How do I write about this?
-     What is an example?  How does one install Cheetah?</para>
- 
- <!-- HERE -->
-    </section>
- 
- 
-    <section id="tutorial-relations">
-     <title>Relations</title>
- 
-     <para>UNFINISHED</para>
- 
-    </section>
- 
-   </chapter>
- 
- 
-   <chapter id="concepts">
-    <title>Overview of &pooma; Concepts</title>
- 
-    <para>Describe the software application layers similar to
-    <filename>papers/SCPaper-95.html</filename> and "Short Tour of
-    &pooma;" in <filename>papers/SiamOO98_paper.ps</filename>.
-    Section&nbsp;2.2, "Why a Framework?," of
-    <filename>papers/pooma.ps</filename> argues why a layered approach
-    eases use.  Section&nbsp;3.1, "Framework Layer Description,"
-    describes the five layers.</para>
- 
-    <para>FINISH: Write short glossary entries for each of these.</para>
- 
-    <para>FINISH: Look through the source code to ensure all main
-    concepts are listed.</para>
- 
-    <para>Here are (preliminary) &pooma; equations:</para>
- 
-    <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	  orient="port" pgwide="0">
-     <title>&pooma; <quote>Equations</quote></title>
- 
-     <tgroup cols="1" align="center">
-      <tbody>
-       <row>
-        <entry><para>field = data + materials + centering + layout + mesh</para></entry>
-        </row>
-       <row>
-        <entry><para>map from space to values</para></entry>
-       </row>
-       <row>
-        <entry><para>array = data + layout</para></entry>
-        </row>
-       <row>
-        <entry><para>map from indices to values</para></entry>
-       </row>
-       <row>
-        <entry><para>mesh = layout + origin + spacings</para></entry>
-        </row>
-       <row>
-        <entry><para>distribute domain through physical space</para></entry>
-       </row>
-       <row>
-        <entry><para>layout = domain + partition + layout_tag (distributed/replicated)</para></entry>
-        </row>
-       <row>
-        <entry><para>distribute domain's blocks among processors/contexts</para></entry>
-       </row>
-       <row>
-        <entry><para>partition = blocks + guard layers</para></entry>
-        </row>
-       <row>
-        <entry><para>split domain into blocks</para></entry>
-       </row>
-       <row>
-        <entry><para>domain = newDomain</para></entry>
-        </row>
-       <row>
-        <entry><para>space of permissible indices</para></entry>
-       </row>
-      </tbody>
-     </tgroup>
-    </table>
- 
- 
-    <para>FINISH: Following is a first try at describing the &pooma;
-    abstraction layers.  See also paper illustration.</para>
- 
-    <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	  orient="port" pgwide="0">
-     <title>&pooma; Abstraction Layers</title>
- 
-     <tgroup cols="1" align="center">
-      <tbody>
-       <row>
-        <entry><para>application program</para></entry>
-       </row>
-       <row>
-        <entry><para>&array; &field; (should have
-        <type>FieldEngine</type> under it)</para></entry>
-       </row>
-       <row>
-        <entry><para>&engine;</para></entry>
-       </row>
-       <row>
-        <entry><para>evaluators</para></entry>
-       </row>
-      </tbody>
-     </tgroup>
-    </table>
- 
-    <para>FINISH: How does parallel execution fit in?</para>
- 
-    <para>FINISH: Should we also name and describe each layer?</para>
- 
-    <section id="concepts-domains">
-     <title>Domains</title>
- 
-     <blockquote>
-      <attribution>Section&nbsp;4 "Future Improvements in
-      &pooma;&nbsp;II" of
-      <filename>papers/SiamOO98_paper.ps</filename></attribution>
-      <para>
-      A &domain; is a set of discrete points in some space.&hellip;
-      &domain;s provide all of the expected domain calculus
-      capabilities such as subsetting and intersection.
-      </para>
-     </blockquote>
- 
-     <para>Section&nbsp;3, "Domains and Views," of
-     <filename>papers/iscope98.pdf</filename> describes five types of
-     domains</para>
-    </section>
- 
- 
-    <section id="concepts-layouts">
-     <title>Layouts</title>
- 
-     <para>UNFINISHED</para>
- 
-     <para>Also describe partitions and guard cells within here.</para>
- 
-    </section>
- 
- 
-    <section id="concepts-meshes">
-     <title>Meshes</title>
- 
-     <para>UNFINISHED</para>
-    </section>
- 
- 
-    <section id="concepts-data_parallel">
-     <title>Data-Parallel Statements</title>
- 
-     <para>Can we use "An Overview of &pete;" from
-     <filename>papers/PETE_DDJ/ddj_article.html</filename> or is this
-     too low-level?</para>
- 
-     <para>Section&nbsp;3.2.1 of <filename>papers/pooma.ps</filename>
-     gives a simple example of data-parallel expression.  It also has a
-     paragraph introducing data-parallel operations and selecting
-     subsets of domains.  Section&nbsp;3.4 describes the Chained
-     Expression Object (<acronym>CEO</acronym>), apparently a precursor
-     of &pete;.  Regardless, it provides some motivation and
-     introductory material.</para>
- 
-     <para>From Section&nbsp;4 of
-     <filename>papers/SiamOO98_paper.ps</filename>:</para>
- 
-     <para>This version of &pete; reduces compile time of user codes
-     and utilizes compile-time knowledge of expression &domain;s for
-     better optimization.  For example, more efficient loops for
-     evaluating an expression can be generated if &pete; knows that the
-     &domain; has unit stride in memory.</para>
- 
-     <para>Section&nbsp;4, "Expressions and Evaluators", of
-     <filename>papers/iscope98.pdf</filename> has a good explanation of
-     &pooma;&nbsp;II's expression trees and expression engines.</para>
- 
-     <para>COMMENT: <filename
- 			     class="libraryfile">background.html</filename> has some related
-     &pete; material.</para>
-    </section>
- 
-    <section id="concepts-containers">
-     <title>Containers</title>
- 
-     <section id="concepts-containers-array">
-      <title>&array;</title>
- 
-      <blockquote>
-       <attribution>Section&nbsp;4 "Future Improvements in
-      &pooma;&nbsp;II" of
-      <filename>papers/SiamOO98_paper.ps</filename></attribution>
-       <para>
-      An &array; can be thought of as a map from one &domain; to
-      another.&hellip; &array;s depend only on the interface of
-      &domain;s.  Thus, a subset of view of an &array; can be
-      manipulated in all the same ways as the original &array;.
-      &array;s can perform indirect addressing because the output
-      &domain; one one &array; can be used as the input &domain; of
-      another &array;.  &array;s also provide individual element
-      access.</para>
-      </blockquote>
- 
- 
-      <para>
-       (unformatted) From
-       <filename>papers/GenericProgramming_CSE/dubois.html</filename>:
-       The &pooma; &array; concept provides an example of how these
-       generic-programming features can lead to flexible and efficient
-       code. An Array maps a fairly arbitrary input domain to an
-       arbitrary range of outputs. When used by itself, an &array;
-       object <varname>A</varname> refers to all of the values in its
-       domain. Element-wise mathematical operations or functions can be
-       applied to an array using straightforward notation, like A + B
-       or sin(A). Expressions involving Array objects are themselves
-       Arrays. The operation A(d), where d is a domain object that
-       describes a subset of A's domain, creates a view of A that
-       refers to that subset of points. Like an array expression, a
-       view is also an Array. If d represents a single point in the
-       domain, this indexing operation returns a single value from the
-       range. Equivalently, one can index an N-dimensional Array by
-       specifying N indices, which collectively specify a single point
-       in the input domain: A(i1, i2, ..., iN).</para>
- 
-      <para>The &pooma; multi-dimensional Array concept is similar to
-       the Fortran 90 array facility, but extends it in several
-       ways. Both &pooma; and Fortran arrays can have up to seven
-       dimensions, and can serve as containers for arbitrary
-       types. Both support the notion of views of a portion of the
-       array, known as array sections in F90. The &pooma; Array concept
-       supports more complex domains, including bounded, continuous
-       (floating-point) domains. Furthermore, Array indexing in &pooma;
-       is polymorphic; that is, the indexing operation X(i1,i2) can
-       perform the mapping from domain to range in a variety of ways,
-       depending on the particular type of the Array being
-       indexed.</para>
- 
-      <para>Fortran arrays are dense and the elements are arranged
-       according to column-major conventions. Therefore, X(i1,i2)
-       refers to element number i1-1+(i2-1)*numberRowsInA. However, as
-       Fig. 1 shows, Fortran-style "Brick" storage is not the only
-       storage format of interest to scientific programmers. For
-       compatibility with C conventions, one might want to use an array
-       featuring dense, row-major storage (a C-style Brick). To save
-       memory, it might be advantageous to use an array that only
-       stores a single value if all its element values are the
-       same. Other sparse storage schemes that only store certain
-       values may also be desirable. To exploit parallelism, it is
-       convenient for an array's storage to be broken up into patches,
-       which can be processed independently by different CPUs. Finally,
-       one can imagine an array with no data at all. For example, the
-       values can be computed from an expression involving other
-       arrays, or analytically from the indices.
-      </para>
- 
-      <para>The &pooma; &array; Class Template</para>
- 
-      <para>Next we describe &pooma;'s model of the Array concept, the
-      Array class template. The three most important requirements from
-      the point of view of overall design are: (1) arbitrary domain,
-      (2) arbitrary range, and (3) polymorphic indexing. These express
-      themselves in the template parameters for the &pooma; Array
-      class. The template
-      <programlisting>
-      template &lt;int Dim, class T = double, class EngineTag = Brick&gt;
-      class Array;
-      </programlisting>
-      is a specification for creating a set of classes all named
-      Array. The template parameters Dim, T, and EngineTag determine
-      the precise type of the Array. Dim represents the dimension of
-      the array's domain. T gives the type of array elements, thereby
-      defining the output range of the array. EngineTag specifies the
-      the manner of indexing and types of the indices.</para>
- 
-      <para>End From
-      <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
- 
-      <para>Section&nbsp;2, "Arrays and Engines," of
-      <filename>papers/iscope98.pdf</filename> describes both &array;s
-      and &engine;s.  This may or may not duplicate the material in
-      <filename>papers/GenericProgramming_CSE/dubois.html</filename>.</para>
- 
-      <section id="concepts-containers-array-views">
-       <title>Views of &array;s</title>
- 
-       <para>Section&nbsp;3, "Domains and Views," of
-       <filename>papers/iscope98.pdf</filename> motivates the need for
-       views:
-        <blockquote>
- 	<para>One of the primary uses of domains is to specify
- 	subsections of &array; objects.  Subarrays are a common
- 	feature of array classes; however, it is often difficult to
- 	make such subarrays behave like first-class objects.  The
- 	&pooma;&nbsp;II engine concept provides a clean solution to
- 	this problem: subsetting an &array; with a domain object
- 	creates a new &array; that has a view engine.</para>
-        </blockquote>
-       </para>
-      </section>
-     </section>
- 
-     <section id="concepts-containers-field">
-      <title>&field;</title>
- 
-      <para>QUESTION: Do we include boundary conditions here?
- 
-      FINISH: Do we have an example that shows something not possible
-      with &array;?</para>
- 
-      <para>Describe and illustrate multi-material and
-      multivalue?</para>
- 
-      <para>ADD: description of meshes and guard layers.</para>
- 
-     </section>
- 
- 
-     <section id="concepts-containers-matrix">
-      <title><type>TinyMatrix</type></title>
- 
-      <para>Section&nbsp;3.2.2 of
-      <filename>papers/pooma.ps</filename> describes &vector;s and
-      matrix classes.</para>
-     </section>
-    </section>
- 
-    <section id="concepts-engines">
-     <title>Engines</title>
- 
-     <para>(unformatted) From
-     <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
- 
-     <para>The Engine Concept</para>
- 
-     <para>To implement polymorphic indexing, the Array class defers
-     data storage and data lookup to an engine object. The requirements
-     that the Array template places on its engine provide the
-     definition for the Engine concept. We'll describe these by
-     examining a simplified version of the Array template, shown in
-     Fig. 2.</para>
- 
-     <para>First, the Array class determines and exports (makes
-     Engine_t part of Array's public interface) the type of the engine
-     class that it will use:
-     <programlisting>
-     typedef Engine&lt;Dim, T, EngineTag&gt; Engine_t;
-     </programlisting>
-     This statement declares Engine_t to be an alias for the type
-     Engine&lt;Dim,T,EngineTag&gt;. This is the first requirement
-     placed on engine classes: they must be specializations of a
-     general Engine template whose template parameters are identical to
-     those of Array. Next, the Array template determines the type of
-     scalar arguments (indices) to be used in operator(), the function
-     that implements &pooma;'s Fortran-style indexing syntax X(i1,i2):
-     <programlisting>
-     typedef typename Engine_t::Index_t Index_t;
-     </programlisting>
-     This statement defines another type alias:
-     Array&lt;Dim,T,EngineTag&gt;::Index_t is simply an alias for
-     Engine_t::Index_t. Engine_t::Index_t is a qualified name, which
-     means that the type Index_t is found in the class Engine_t. This
-     is the second requirement for the Engine concept: the class
-     Engine_t must define a public type called Index_t. This line will
-     not compile if that definition is not supplied. This indirection
-     is one of the ways that &pooma; supports polymorphic indexing. If
-     the Engine works with a discrete integer domain, it defines its
-     Index_t to be an integral type. If the Engine works in a
-     continuous domain, it defines its Index_t to be a floating-point
-     type.</para>
- 
-     <para>The data lookup is performed in the operator() function. We
-     see that Array simply passes the indices on to its engine
-     object. Thus, we have the third requirement for the Engine
-     concept: it must provide a version of operator() that takes Dim
-     values of type Index_t.</para>
- 
-     <para>Simply passing the indices on to the engine object may seem
-     odd. After all, engine(i,j) looks like we're just indexing another
-     array. There are several advantages to this extra level of
-     indirection. The Array class is as faithful a model of the Array
-     concept as possible, while the Engine class is a low-level
-     interface to a user-defined data source. As a result, Array has a
-     wide variety of constructors for user convenience, while engines
-     have but a few. Array supports a wide variety of overloaded
-     operator() functions for view creation and indexing. Engines
-     support indexing only. Array does not have direct access to the
-     data, which is managed by the engine object. Finally, Array has a
-     wide variety of overloaded mathematical operators and functions,
-     and works with the Portable Expression Template Engine (PETE) [4]
-     to provide efficient evaluation of Array expressions. Engines have
-     no such support. In general, Array is much more complex and
-     feature-laden than Engine. This is the prime advantage of the
-     separation of interface and implementation: Array only has to be
-     implemented once by the &pooma; developers. Engines are simple
-     enough to be written by users and plugged directly into the Array
-     framework.</para>
- 
-     <para>Figure 3 illustrates the "Brick" specialization of the
-     Engine template, which implements Fortran-style lookup into a
-     block of memory. First, there is the general Engine template,
-     which is empty as there is no default behavior for an unknown
-     EngineTag. The general template is therefore not a model for the
-     Engine concept and Array classes attempting to use it will not
-     compile. Next, there is the definition of the Brick class, a
-     policy tag whose sole purpose is to select a particular
-     specialization of the Engine template. Finally, there is the
-     partial specialization of the Engine template. Examining its body,
-     we see the required Index_t typedef and the required operator(),
-     which follows the Fortran prescription for generating an offset
-     into the data block based on the row, column, and the number of
-     rows. All of the requirements are met, so the Brick-Engine class
-     is a model of the Engine concept.</para>
- 
-     <para>End From
-     <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
- 
-     <para>(unformatted) From
-     <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
- 
-     <para>Compile-time Versus Run-Time Polymorphism</para>
- 
-     <para>Encapsulating the indexing in an Engine class has important
-     advantages, both in terms of flexibility and efficiency. To
-     illustrate this point, we introduce the PolarGaussian-Engine
-     specialization in Fig. 4. This is an analytic engine that
-     calculates its values directly from its inputs. Unlike the
-     Brick-Engine, this engine is "indexed" with data of the same type
-     as its output: it maps a set of T's to a single T. Therefore, the
-     Index_t typedef selects T as the index type, as opposed to the int
-     in the Brick-Engine specialization. The operator() function also
-     differs in that it computes the return value according to an
-     analytic formula.</para>
- 
-     <para>Both Engine&lt;Dim,T,Brick&gt; and
-     Engine&lt;Dim,T,PolarGaussian&gt; can be plugged in to an Array by
-     simply varying the Array's EngineTag. This is possible despite the
-     fact that the two classes exhibit dramatically different behavior
-     because they are both models of the Engine concept.</para>
- 
-     <para>Notice that we have achieved polymorphic indexing without
-     the use of inheritance or virtual functions. For instance,
-     consider the following code snippet:
-     <programlisting>
-     Array<2, double, Brick> a;
-     Array<2, double, PolarGaussian> b;
- 
-     double x = a(2, 3);     // x = a.engine.data[2 + 3 * a.engine.numRows];
-     double y = b(2.0, 3.0); // y = exp(-(2.0*2.0+3.0*3.0) / b.engine.delta);
-     </programlisting>
-     The data lookup functions for the two Arrays perform completely
-     different operations. Since this is accomplished using static
-     types, it is known as compile-time polymorphism. Moreover,
-     everything is known at compile time, so the functions are fully
-     inlined and optimized, thereby yielding code equivalent to that
-     shown in the comments above.</para>
- 
-     <para>The flexibility and efficiency of compile-time polymorphism
-     cannot be duplicated with a run-time implementation. To illustrate
-     this point, in Fig. 5, we re-implement our Array concept using the
-     classic Envelope-Letter pattern [5], with the array class,
-     RTArray, being the envelope and the run-time-engine, RTEngine,
-     being the letter. RTArray defers data lookup to the engine object
-     by invoking the engine's functions through a pointer to the
-     RTEngine base class. Figure 6 illustrates the RTEngine base class
-     and Fig. 7 illustrates two descendants: RTBrick and
-     RTPolarGaussian.</para>
- 
-     <para>The run-time implementation provides the same basic
-     functionality as the compile-time implementation, but it is not as
-     flexible or as efficient. It lacks flexibility in that the return
-     type of the indexing operation must be specified in the RTEngine
-     base class and in the RTArray class. Thus, in Figs. 5 and 6,we see
-     versions of RTArray::operator() and RTEngine::index functions that
-     take both int's and T's. If the programmer wants to add another
-     index-type option, these classes must be modified. This is a
-     violation of the open-closed principle proposed by Meyer
-     [6]. Also, since RTEngine descendants will usually only implement
-     one version of index, we cannot make RTEngine an abstract base
-     class. Instead, we have the default versions of index throw an
-     exception. Thus, compile-time error checking is
-     weakened. Furthermore, since indexing is done via a virtual
-     function call, it will almost never be inlined, which is not
-     acceptable in most scientific applications.</para>
- 
-     <para>There are advantages to the Envelope-Letter approach. First,
-     all RTArray objects have the same type, allowing them to be stored
-     in homogeneous collections. This can simplify the design of some
-     applications. Second, RTArray objects can change their engines at
-     runtime, and thus effectively change their types on the fly??this
-     is the primary reason for using the Envelope-Letter idiom, and can
-     be very important in some applications.</para>
- 
-     <para>For most scientific applications, however, these issues are
-     minor, and maximum performance for array indexing is of paramount
-     importance. Our compile-time approach achieves this performance
-     while providing the desired polymorphic indexing.</para>
- 
-     <para>From Section&nbsp;4 of
-     <filename>papers/SiamOO98_paper.ps</filename>:</para>
- 
-     <para>The &array; class is templated on an &engine; type that
-     handles the actual implementation of the mapping from input to
-     output.  Thus, the &array; interface features are completely
-     separate from the implementation, which could be a single &c;
-     array, a function of some kind or some other mechanism.  This
-     flexibility allows an expression itself to be viewed through the
-     &array; interface.  Thus, one can write something like
-     <programlisting>
-     foo(A*B+C);
-     </programlisting> where <varname>A</varname>, <varname>B</varname> and
-     <varname>C</varname> are &array;s and <function>foo</function> is
-     a function taking an &array; as an argument.  The expression
-     <statement><varname>A</varname>*<varname>B</varname>+<varname>C</varname></statement>
-     will only be evaluated by the expression engine as needed by
-     <function>foo</function>.</para>
- 
-     <para>In fact, one can even write &engine;s which are wrappers
-     around external data structures created in non-&pooma; codes and
-     know to manipulate these structures.  Once this is done, the
-     external entities have access to the entire &array; interface and
-     can utilize all of the powerful features of
-     &pooma;&nbsp;II.</para>
- 
-     <para>Section&nbsp;2, "Arrays and Engines," of
-     <filename>papers/iscope98.pdf</filename> describes both &array;s
-     and &engine;s.  This may or may not duplicate the material in
-     <filename>papers/GenericProgramming_CSE/dubois.html</filename>.</para>
- 
-     <para>Section&nbsp;4, "Expressions and Evaluators", of
-     <filename>papers/iscope98.pdf</filename> has a good explanation of
-     &pooma;&nbsp;II's expression trees and expression engines.</para>
- 
-     <variablelist>
-      <varlistentry><term><type>MultiPatch</type> Engine</term>
-       <listitem><para>From <filename
-       class="libraryfile">README</filename>: To actually use multiple
-       contexts effectively, you need to use the MultiPatch engine with
-       patch engines that are Remote engines.  Then the data will be
-       distributed across multiple contexts instead of being copied on
-       every context.  See the files in example/Doof2d for a simple
-       example that creates a MultiPatch array that can be distributed
-       across multiple contexts and performs a stencil computation on
-       that array.</para></listitem>
-      </varlistentry>
-     </variablelist>
-    </section>
- 
- 
-    <section id="concepts-relations">
-     <title>Relations</title>
- 
-     <para>UNFINISHED</para>
-    </section>
- 
- 
-    <section id="concepts-stencils">
-     <title>Stencils</title>
- 
-     <para>Section&nbsp;3.5.4, "Stencil Objects," of
-     <filename>papers/pooma.ps</filename> provides a few uses of
-     stencils.</para>
- 
-     <para>Section&nbsp;5, "Performance," of
-     <filename>papers/iscope98.pdf</filename> motivates and explains
-     stencils.</para>
-    </section>
- 
- 
-    <section id="concepts-contexts">
-     <title>Contexts</title>
- 
-     <blockquote>
-      <attribution><filename
-      class="libraryfile">background.html</filename></attribution>
-      <para>In order to be able to cope with the variations in machine
-      architecture noted above, &pooma;'s distributed execution model
-      is defined in terms of one or more contexts, each of which may
-      host one or more threads. A context is a distinct region of
-      memory in some computer. The threads associated with the context
-      can access data in that memory region and can run on the
-      processors associated with that context. Threads running in
-      different contexts cannot access memory in other contexts.</para>
- 
-      <para>A single context may include several physical processors,
-      or just one. Conversely, different contexts do not have to be on
-      separate computers&mdash;for example, a 32-node SMP computer could
-      have up to 32 separate contexts. This release of &pooma; only
-      supports a single context for each application, but can use
-      multiple threads in the context on supported platforms. Support
-      for multiple contexts will be added in an upcoming
-      release.</para>
-     </blockquote>
-    </section>
- 
- 
-    <section id="concepts-utilities">
-     <title>Utility Types: ???TITLE??</title>
- 
-     <section id="concepts-containers-vector">
-      <title>&vector;</title>
- 
-      <para>Section&nbsp;3.2.2 of
-      <filename>papers/pooma.ps</filename> describes &vector;s and
-      matrix classes.</para>
-     </section>
- 
-    </section>
-   </chapter>
- 
- 
-   <chapter id="sequential">
-    <title>Writing Sequential Programs</title>
- 
-    <para>UNFINISHED</para>
- 
-    <section id="sequential-benchmarks">
-     <title>&benchmark; Programs</title>
- 
-     <para>Define a &benchmark; program vs. an example or an
-     executable.  Provide a short overview of how to run these
-     programs.  Provide an overview of how to write these programs.
-     See <filename
- 		  class="headerfile">src/Utilities/Benchmark.h</filename>.</para>
-    </section>
- 
- 
-    <section id="sequential-inform">
-     <title>Using <type>Inform</type>s for Output</title>
- 
-     <para>UNFINISHED</para>
-    </section>
- 
-   </chapter>
- 
- 
-   <chapter id="parallel">
-    <title>Writing Distributed Programs</title>
- 
-    <para>Discuss the distributed model and guard cells.  See <filename
-    class="libraryfile">docs/parallelism.html</filename>.</para>
- 
-    <para>Does any of the parallel implementation described in
-    <filename>papers/SCPaper-95.html</filename> still apply?</para>
- 
-    <para>?Tuning program for maximize parallel performance?</para>
- 
-    <para>external references to &mpi; and threads</para>
- 
-    <para>QUESTION: Are there interesting, short parallel programs in
-    any &mpi; book that we can convert to &pooma;?</para>
- 
-    <section>
-     <title>Layouts</title>
- 
-     <para>An out-of-date description can be found in Section&nbsp;3.3,
-     especially 3.3.2, of <filename>papers/pooma.ps</filename>
-     describes the global/local interactions and parallel abstraction
-     layers.</para>
-    </section>
- 
-    <section>
-     <title>Parallel Communication</title>
- 
-     <para>An out-of-date description can be found in
-      Section&nbsp;3.3.3 of <filename>papers/pooma.ps</filename></para>
-    </section>
- 
-    <section>
-     <title>Using Threads</title>
- 
-     <para>QUESTION: Where do threads fit into the manual?  Do threads
-     even work?</para>
- 
-     <para>From Section&nbsp;4, of
-     <filename>papers/SiamOO98_paper.ps</filename></para>
- 
-     <para>&pooma;&nbsp;II will make use of a new parallel run-time
-     system called &smarts; that is under development at the ACL.
-     &smarts; supports lightweight threads, so the evaluator will be
-     able to farm out data communication tasks and the evaluation of
-     subsets of an expression to multiple threads, thus increasing the
-     overlap of communication and computation.  Threads will also be
-     available at the user level for situations in which a
-     task-parallel approach is deemed appropriate.</para>
-    </section>
- 
-   </chapter>
- 
- 
-   <chapter id="under_the_hood">
-    <title>Under the Hood: How &pooma; Works</title>
- 
-    <para>from point of view of &cc; interpreter</para>
- 
-    <section id="under_the_hood-PETE">
-     <title>&pete;</title>
- 
-     <para>Use the material in
-     <filename>papers/PETE_DDJ/ddj_article.html</filename>, which gives
-     example code and descriptions of how the code works.</para>
- 
-     <para>See material in <filename
-     class="libraryfile">background.html</filename>'s <quote>Expression
-     Templates.</quote></para>
-    </section>
- 
-   </chapter>
- 
- 
-   <chapter id="debugging_profiling">
-    <title>Debugging and Profiling &pooma; Programs</title>
- 
-    <para>UNFINISHED</para>
- 
-   </chapter>
- 
- 
-   <chapter id="example">
-    <title>Example Program: Jacobi Solver</title>
- 
-    <para>QUESTION: Is this chapter necessary?  Do we have enough
-    existing source code to write this chapter?</para>
- 
-   </chapter>
-  </part>
- 
-  <part id="reference">
-   <title>&pooma; Reference Manual</title>
- 
-   <chapter id="tmp_ref">
-    <title>TMP: This Chapter Holds These Comments But Will Be Removed</title>
- 
-    <para>For each template parameter need to describe the constraints
-    on it.</para>
- 
-    <para>Remove this section when the following concerns have been
-    addressed.</para>
- 
-    <para>Add a partintro explaining file suffixes such as <filename
-    class="headerfile">.h</filename>, <filename
-    class="libraryfile">.cpp</filename>, <filename
-    class="libraryfile">.cmpl.cpp</filename>, <filename
-    class="libraryfile">.mk</filename>, <filename
-    class="libraryfile">.conf</filename>.  Should we also explain use
-    of <literal>inline</literal> even when necessary and the template
-    model, <!-- FIXME: s/literal/keyword/ --> e.g., including <filename
-    class="libraryfile">.cpp</filename> files.</para>
- 
-    <para>QUESTION: What are the key concepts around which to organize
-    the manual?</para>
- 
-    <para>QUESTION: What format should the manual use?</para>
- 
-    <blockquote>
-     <attribution>Musser, Derge, and Sanai, &sect;20.0.</attribution>
-     <para>It is important to state the requirements on the components
-     as generally as possible.  For example, instead of saying
-     <quote>class <type>X</type> must define a member function
-     <function>operator++()</function>,</quote> we say <quote>for any
-     object&nbsp;<varname>x</varname> of type&nbsp;<type>X</type>,
-     <classname>++x</classname> is defined.</quote></para>
-    </blockquote>
-   </chapter>
- 
- 
-   <chapter id="class_structure_ref">
-    <title>A Typical &pooma; Class</title>
- 
-    <variablelist>
-     <title>Class Member Notation</title>
-     <varlistentry>
-      <term><fieldsynopsis>
-        <varname>*_t</varname>
-       </fieldsynopsis>
-      </term>
-      <listitem>
-       <para>type within a class.  QUESTION: What is the &cc; name for
-       this?</para>
-      </listitem>
-     </varlistentry>
-     <varlistentry>
-      <term><fieldsynopsis>
-        <varname>*_m</varname>
-       </fieldsynopsis>
-      </term>
-      <listitem>
-       <para>data member</para>
-      </listitem>
-     </varlistentry>
-    </variablelist>
- 
-    <variablelist>
-     <title>&pooma; Class Vocabulary</title>
-     <varlistentry>
-      <term>component</term>
-      <listitem>
-       <para>one of several values packaged together.  For example, a
-       three-dimensional vector has three components, i.e., three
-       values.</para>
-      </listitem>
-     </varlistentry>
-     <varlistentry>
-      <term>element-wise</term>
-      <listitem>
-       <para>applied to each element in the group, e.g., an array</para>
-      </listitem>
-     </varlistentry>
-     <varlistentry>
-      <term>reduction</term>
-      <listitem>
-       <para>repeated application of a binary operator to all elements,
-       yielding one value</para>
-      </listitem>
-     </varlistentry>
-     <varlistentry>
-      <term>tag</term>
-      <listitem>
-       <para>an enumerated value indicating inclusion in a particular
-       semantic class.  The set of values need not be explicitly
-       declared.</para>
-      </listitem>
-     </varlistentry>
-    </variablelist>
- 
-   </chapter>
- 
- 
-   <chapter id="installing_configuring_ref">
-    <title>Installing and Configuring &pooma;</title>
- 
-    <orderedlist>
-     <listitem>
-      <para>Installing &pooma;.</para>
-     </listitem>
-     <listitem>
-      <para>Requirements for configuration files.</para>
-     </listitem>
-    </orderedlist>
- 
-    <para>Include descriptions of using &smarts;, &cheetah;, &tau;,
-    &pdt;.</para>
- 
-    <para>QUESTION: Does it install on windows and on mac?  If so, what
-    are the instructions?  See also <filename
-    class="libraryfile">INSTALL.{mac,unix,windows}.</filename></para>
- 
-    <para><filename class="libraryfile">README</filename> has some
-    information on &cheetah; and threads in the <quote>Message-Based
-    Parallelism</quote> section.</para>
- 
-    <para>Which additional packages are necessary and when?</para>
- 
-    <para>What configure options should we list?  See <filename
-    class="libraryfile">configure</filename>.  Be sure to list
-    debugging option and how its output relates to <filename
-    class="libraryfile">config/LINUXgcc.suite.mk</filename>.</para>
- 
-    <para><filename class="directory">config/arch</filename> has files
-    for (OS, compiler) pairs.  Explain how to modify a configuration
-    file.  List requirements when making a new configuration file (low
-    priority).</para>
- 
-    <para><filename
-    class="libraryfile">config/LINUXgcc.suite.mk</filename> has output
-    from <filename class="libraryfile">configure</filename>.  Useful to
-    relate to configuration files and <filename
-    class="libraryfile">configure</filename>'s debugging output.</para>
-    
-   </chapter>
- 
- 
-   <chapter id="compilation_ref">
-    <title>Compilation and &make; Files</title>
- 
-    <para>We assume Gnu make.  Do we know what assumptions are made?</para>
- 
-    <para>How do all these files interact with each other?  Ala a make
-    interpreter, give an example of which files are read and
-    when.</para>
- 
-    <variablelist>
-     <varlistentry><term><filename
-     class="libraryfile">config/Shared/README.make</filename></term>
-     <listitem><para>This has short descriptions of many files,
-     especially in <filename
-     class="directory">config/Shared</filename>.</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">makefile</filename></term>
-     <listitem><para>These appear throughout all directories.  What are
-     the equivalences classes and what are their
-     parts?</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">include.mk</filename></term>
-     <listitem><para>What does this do?  Occurs in many directories:
-     when?  Template seems to be <filename
-     class="libraryfile">config/Shared/include2.mk</filename>.</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">subdir.mk</filename></term>
-     <listitem><para>list of subdirectories; occurs in several
-     directories: when?  <filename
-     class="libraryfile">src/subdir.mk</filename> is a good
-     example.</para></listitem>
-     </varlistentry>
-     <varlistentry>
-      <term><filename class="libraryfile">objfile.mk</filename></term>
-      <listitem>
-       <para>list of object files to construct, presumably from
-       <filename>*.cmpl.cpp</filename> files.
-       <filename>src/Utilities/objfile.mk</filename> is an
-       example.</para>
-      </listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">config/Shared/rules.mk</filename></term>
-     <listitem><para>most compiler rules</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">config/head.mk</filename></term>
-     <listitem><para>read at beginning of each
-     makefile?</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">config/Shared/tail.mk</filename></term>
-     <listitem><para>read at end of each makefile?</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">config/Shared/variables.mk</filename></term>
-     <listitem><para>Is this used?</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">config/Shared/compilerules.mk</filename></term>
-     <listitem><para>table of origin and target suffixes and commands
-     for conversion</para></listitem>
-     </varlistentry>
- 
-    </variablelist>
- 
-   </chapter>
- 
- 
-   <chapter id="arrays_ref">
-    <title>&array;s</title>
- 
-    <para>Include <filename
- 			   class="headerfile">src/Pooma/Arrays.h</filename> to use &array;s.
-    The implementation source code is in <filename
- 						  class="directory">src/Array</filename>.</para>
- 
-    <para>FINISH: Define an array.  Introduce its parts.</para>
- 
-    <para>ADD: some mention of the maximum supported number of
-    dimensions somewhere.</para>
- 
-    <section id="arrays_ref-array">
-     <title>The &array; Container</title>
- 
-     <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	   orient="port" pgwide="0">
-      <title>Template Parameters</title>
- 
-      <tgroup cols="2" align="left">
-       <thead>
-        <row>
- 	<entry>Parameter</entry>
- 	<entry>Interpretation</entry>
-        </row>
-       </thead>
-       <tbody>
-        <row>
- 	<entry><varname>Dim</varname></entry>
- 	<entry><para>dimension</para></entry>
-        </row>
-        <row>
- 	<entry><varname>T</varname></entry>
- 	<entry><para>array element type</para></entry>
-        </row>
-        <row>
- 	<entry>EngineTag</entry>
- 	<entry><para>type of computation engine object</para></entry>
-        </row>
-       </tbody>
-      </tgroup>
-     </table>
- 
-     <para>QUESTION: How do I introduce class type definitions, when
-     they are used, i.e., compile-time or run-time, and when
-     programmers should use them?</para>
- 
-     <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	   orient="port" pgwide="0">
-      <title>Compile-Time Types and Values</title>
- 
-      <tgroup cols="2" align="left">
-       <thead>
-        <row>
- 	<entry>Type or Value</entry>
- 	<entry>Interpretation</entry>
-        </row>
-       </thead>
-       <tbody>
-        <row>
- 	<entry><fieldsynopsis><varname>This_t</varname></fieldsynopsis></entry>
- 	<entry><para>the &array; object's type</para></entry>
-        </row>
-        <row>
- 	<entry><fieldsynopsis><varname>Engine_t</varname></fieldsynopsis></entry>
- 	<entry><para>the &array; object's engine's type</para></entry>
-        </row>
-        <row>
- 	<entry><fieldsynopsis><varname>EngineTag_t</varname></fieldsynopsis></entry>
- 	<entry><para>indication of engine's category</para></entry>
-        </row>
-        <row>
- 	<entry><fieldsynopsis><varname>Element_t</varname></fieldsynopsis></entry>
- 	<entry><para>the type of the array elements, i.e., <type>T</type></para></entry>
-        </row>
-        <row>
- 	<entry><fieldsynopsis><varname>ElementRef_t</varname></fieldsynopsis></entry>
- 	<entry><para>the type of a reference to an array element,
- 	i.e., <type>T&</type>.  Equivalently, the type to write to a
- 	single element.</para></entry>
-        </row>
-        <row>
- 	<entry><fieldsynopsis><varname>Domain_t</varname></fieldsynopsis></entry>
- 	<entry><para>the array's domain's type, i.e., the type of the
- 	union of all array indices</para></entry>
-        </row>
-        <row>
- 	<entry><fieldsynopsis><varname>Layout_t</varname></fieldsynopsis></entry>
- 	<entry><para>unknown</para></entry>
-        </row>
-        <row>
- 	<entry><fieldsynopsis><varname>dimensions</varname></fieldsynopsis></entry>
- 	<entry><para>integer equalling the number of dimensions, i.e.,
- 	<varname>Dim</varname></para></entry>
-        </row>
-        <row>
- 	<entry><fieldsynopsis><varname>rank</varname></fieldsynopsis></entry>
- 	<entry><para>integer equalling the number of dimensions, i.e.,
- 	<varname>Dim</varname>; a synonym for <fieldsynopsis>
- 	   <varname>dimensions</varname></fieldsynopsis></para></entry>
-        </row>
-       </tbody>
-      </tgroup>
-     </table>
- 
-     <section id="arrays_ref-array-constructors">
-      <title>Constructors and Destructors</title>
- 
-      <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	    orient="port" pgwide="0">
-       <title>Constructors and Destructors</title>
- 
-       <tgroup cols="2" align="left">
-        <thead>
- 	<row>
- 	 <entry>Function</entry>
- 	 <entry>Effect</entry>
- 	</row>
-        </thead>
-        <tbody>
- 	<row>
- 	 <entry>
-           <constructorsynopsis>
- 	   <methodname>Array</methodname>
- 	   <void></void>
- 	  </constructorsynopsis>
-          </entry>
- 	 <entry><para>Creates an array that will be resized
- 	 later.</para></entry>
- 	</row>
- 	<row>
- 	 <entry>
-           <constructorsynopsis>
- 	   <methodname>Array</methodname>
- 	   <methodparam><modifier>const</modifier> <type>Engine_t&</type>
- 	    <parameter>engine</parameter></methodparam>
- 	  </constructorsynopsis>
-          </entry>
- 	 <entry><para>Creates an array with an engine equivalent to
-          the <parameter>engine</parameter>.  This array will have the
-          same values as <parameter>engine</parameter>.  QUESTION: Why
-          would a user every want to use this
-          constructor?</para></entry>
- 	</row>
- 	<row>
- 	 <entry>
-           <constructorsynopsis>
- 	   <methodname>Array</methodname>
- 	   <methodparam>
- 	    <modifier>const</modifier>
- 	    <type> Engine&lt;Dim2, T2, EngineTag2&gt;&</type>
- 	    <parameter>engine</parameter>
- 	   </methodparam>
- 	   <methodparam>
- 	    <modifier>const</modifier>
- 	    <type>Initializer&</type> <parameter>init</parameter>
- 	   </methodparam>
- 	  </constructorsynopsis>
-          </entry>
- 	 <entry><para>What does this do?</para></entry>
- 	</row>
- 	<row>
- 	 <entry>ADD ALL CONSTRUCTORS AND DESTRUCTORS.</entry>
- 	</row>
-        </tbody>
-       </tgroup>
-      </table>
-     </section>
- 
- 
-     <section id="arrays_ref-array-initializers">
-      <title>Initializers</title>
- 
-      <para>Add a table.</para>
-     </section>
- 
- 
-     <section id="arrays_ref-array-access">
-      <title>Element Access</title>
- 
-      <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	    orient="port" pgwide="0">
-       <title>&array; Element Access</title>
- 
-       <tgroup cols="2" align="left">
-        <thead>
- 	<row>
- 	 <entry>Function</entry>
- 	 <entry>Effect</entry>
- 	</row>
-        </thead>
-        <tbody>
- 	<row>
- 	 <entry>
-           <methodsynopsis>
- 	   <type>Element_t</type> <methodname>read</methodname>
- 	   <void></void>
- 	  </methodsynopsis>
-          </entry>
- 	 <entry><para>unknown: See line 1839.</para></entry>
- 	</row>
- 	<row>
- 	 <entry>
-           <methodsynopsis>
- 	   <type>Element_t</type> <methodname>read</methodname>
- 	   <methodparam>
- 	    <modifier>const</modifier>
- 	    <type>Sub1&</type> <parameter>s1</parameter>
- 	   </methodparam>
- 	   <methodparam>
- 	    <modifier>const</modifier>
- 	    <type> Sub2&</type> <parameter>s2</parameter>
- 	   </methodparam>
- 	  </methodsynopsis>
-          </entry>
- 	 <entry><para>How does the version with template parameters,
-          e.g., <type>Sub1</type> differ from the <type>int</type>
-          version?</para></entry>
- 	</row>
- 	<row>
- 	 <entry>
-           <methodsynopsis>
- 	   <type>Element_t</type> <methodname>operator()</methodname>
- 	   <methodparam>
- 	    <modifier>const</modifier>
- 	    <type>Sub1&</type> <parameter>s1</parameter>
- 	   </methodparam>
- 	   <methodparam>
- 	    <modifier>const</modifier>
- 	    <type>Sub2&</type> <parameter>s2</parameter>
- 	   </methodparam>
- 	  </methodsynopsis>
-          </entry>
- 	 <entry><para>How does this differ from <statement>read(const
-           Sub1& s1, const Sub2& s2)</statement>?  </para></entry>
- 	</row>
- 	<row>
- 	 <entry>ADD ALL <function>read</function>s and
- 	 <function>operator()</function>s.</entry>
- 	</row>
-        </tbody>
-       </tgroup>
-      </table>
-     </section>
- 
- 
-     <section id="arrays_ref-array-component">
-      <title>Component Access</title>
- 
-      <para>When an array stores elements having components, e.g., an
-      array of vectors, tensors, or arrays, the
-      <methodname>comp</methodname> returns an array consisting of the
-      specified components.  The original and component array share the
-      same engine so changing the values in one affects values in the
-      other.</para>
- 
-      <para>For example, if &n; &times; &n; array <varname>a</varname>
-      consists of three-dimensional real-valued vectors,
-      <statement>a.comp(1)</statement> returns a &n; &times; &n;
-      real-valued array of all the middle vector components.  Assigning
-      to the component array will also modify the middle components of
-      the vectors in <varname>a</varname>.</para>
- 
-      <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	    orient="port" pgwide="0">
-       <title>&array; Component Access</title>
- 
-       <tgroup cols="2" align="left">
-        <thead>
- 	<row>
- 	 <entry>Function</entry>
- 	 <entry>Effect</entry>
- 	</row>
-        </thead>
-        <tbody>
- 	<row>
- 	 <entry>
-           <methodsynopsis>
- 	   <type>UNKNOWN compute this</type> <methodname>comp</methodname>
- 	   <methodparam>
- 	    <modifier>const</modifier>
- 	    <type>int&</type>
- 	    <parameter>i1</parameter>
- 	   </methodparam>
- 	  </methodsynopsis>
-          </entry>
- 	 <entry><para>unknown: See line 1989.</para></entry>
- 	</row>
- 	<row>
- 	 <entry>ADD ALL <methodname>comp</methodname>s.</entry>
- 	</row>
-        </tbody>
-       </tgroup>
-      </table>
-     </section>
- 
-     <section id="arrays_ref-array-accessors">
-      <title>Accessors</title>
- 
-      <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	    orient="port" pgwide="0">
-       <title>&array; Accessor Methods</title>
- 
-       <tgroup cols="2" align="left">
-        <thead>
- 	<row>
- 	 <entry>Function</entry>
- 	 <entry>Effect</entry>
- 	</row>
-        </thead>
-        <tbody>
- 	<row>
- 	 <entry>
-           <methodsynopsis>
- 	   <type>int</type> <methodname>first</methodname>
- 	   <methodparam>
- 	    <type>int</type>
- 	    <parameter>d</parameter>
- 	   </methodparam>
- 	  </methodsynopsis>
-          </entry>
- 	 <entry><para>unknown: See line 2050</para></entry>
- 	</row>
- 	<row>
- 	 <entry>ADD ALL other accessor methods, including
- 	 <methodname>engine</methodname>.</entry>
- 	</row>
-        </tbody>
-       </tgroup>
-      </table>
-     </section>
- 
- 
-     <section id="arrays_ref-array-copying">
-      <title>Copying &array;s</title>
- 
-      <para>Explain how copied arrays and views of arrays share the
-      same underlying engine so changing values in one also affects the
-      other.  This is called a shallow copy.</para>
-     </section>
- 
- 
-     <section id="arrays_ref-array-utilities">
-      <title>Utility Methods</title>
- 
-      <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	    orient="port" pgwide="0">
-       <title>&array; Utility Methods</title>
- 
-       <tgroup cols="2" align="left">
-        <thead>
- 	<row>
- 	 <entry>Function</entry>
- 	 <entry>Effect</entry>
- 	</row>
-        </thead>
-        <tbody>
- 	<row>
- 	 <entry>
-           <methodsynopsis>
- 	   <type>void</type> <methodname>makeOwnCopy</methodname>
- 	   <void></void>
- 	  </methodsynopsis>
-          </entry>
- 	 <entry><para>unknown: See line 2044</para></entry>
- 	</row>
- 	<row>
- 	 <entry>ADD ALL other utility methods.</entry>
- 	</row>
-        </tbody>
-       </tgroup>
-      </table>
-     </section>
- 
-     
-     <section id="arrays_ref-array-implementation">
-      <title>Implementation Details</title>
- 
-      <para>As a container, an &array;'s implementation is quite
-      simple.  Its <keywordname>private</keywordname>data consists of
-      an engine, and it has no <keywordname>private</keywordname>
-      functions.</para>
- 
-      <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	    orient="port" pgwide="0">
-       <title>&array; Implementation Data</title>
- 
-       <tgroup cols="2" align="left">
-        <thead>
- 	<row>
- 	 <entry>Data Member</entry>
- 	 <entry>Meaning</entry>
- 	</row>
-        </thead>
-        <tbody>
- 	<row>
- 	 <entry>
-           <fieldsynopsis>
- 	   <modifier>private</modifier>
- 	   <type>Engine_t</type> <varname>engine_m</varname>
- 	  </fieldsynopsis>
-          </entry>
- 	 <entry><para>engine computing the array's values</para></entry>
- 	</row>
-        </tbody>
-       </tgroup>
-      </table>
- 
-     </section>
-    </section>
- 
- 
-    <section id="arrays_ref-dynamicarray">
-     <title>&dynamicarray;s: Dynamically-Sized Domains</title>
- 
-     <para>A DynamicArray is a read-write array with extra
-     create/destroy methods.  It can act just like a regular Array, but
-     can have a dynamically-changing domain.  See <filename
- 							   class="headerfile">src/DynamicArray/DynamicArray.h</filename>.</para>
- 
-     <para>ADD: Briefly describe what the class does and an example of
-     where it is used.</para>
- 
-     <para>ADD: Check that its interface is actually the same as for
-     &array;.</para>
- 
-     <para>ADD: Check that the operations on dynamic arrays are
-     actually the same as for &array;.  See <filename
- 						     class="headerfile">src/DynamicArray/DynamicArrayOperators.h</filename>,
-     <filename
- 	      class="headerfile">src/DynamicArray/PoomaDynamicArrayOperators.h</filename>,
-     and <filename
- 		  class="headerfile">src/DynamicArray/VectorDynamicArrayOperators.h</filename>.</para>
- 
-     
-     <section id="arrays_ref-dynamicarray-implementation">
-      <title>Implementation Details</title>
- 
-      <para><type>DynamicArray</type> has no
-      <keywordname>protected</keywordname> or
-      <keywordname>private</keywordname> members.</para>
-     </section>
-    </section>
- 
- 
-    <section id="arrays_ref-views">
-     <title>Views of &array;s</title>
- 
-     <para>UNFINISHED</para>
-    </section>
- 
- 
-    <section id="arrays_ref-assignment">
-     <title>&array; Assignments</title>
- 
-     <para>&pooma; supports assignments to &array;s of other &array;s
-     and scalar values.  QUESTION: Is the following correct?  For the
-     former, the right-hand side array's domain must be at least as
-     large as the left-hand side array's domain.  Corresponding values
-     are copied.  Assigning a scalar value to an array ensures all the
-     array elements have the same scalar value.</para>
- 
-     <para>UNFINISHED: Add a table containing assignment operators
-     found one lines 2097&ndash;2202.</para>
-    </section>
- 
- 
-    <section id="arrays_ref-print">
-     <title>Printing &array;s</title>
- 
-     <para>&array;s support output to but not input from IO streams.
-     In particular, output to <type>ostream</type>s and file streams is
-     supported.</para>
- 
-     <para>Add a table, using <filename
-     class="headerfile">src/Array/Array.h</filename>, lines
-     2408&ndash;2421.  See the implementation in <filename
-     class="headerfile">src/Array/PrintArray.h</filename>.</para>
- 
-     <para>QUESTION: How does one print a &dynamicarray;.</para>
-    </section>
- 
- 
-    <section>
-     <title>Expressions Involving &array;s</title>
- 
-     <para>In &pooma;, expressions may contain entire &array;s.  That
-     is, &array;s are first-class objects with respect to expressions.
-     For example, given &array;s <varname>a</varname> and
-     <varname>b</varname>, the expression <statement>a + b</statement>
-     is equivalent to an array containing the element-wise sum of the
-     two arrays.</para>
- 
-     <para>Any finite number of the operators listed below can be used
-     in an expression.  The precedence and order of operation is the
-     same as with ordinary built-in types.</para>
- 
-     <para>QUESTION: Do &field;s also support the same set of
-     operations?</para>
- 
-     <para>QUESTION: Some operations in <filename
- 						 class="headerfile">src/Field/FieldOperators.h</filename> use both
-     &array; and &field;.  Do we list them here or in the &field;
-     section or both or somewhere else?</para>
- 
-     <para>In the table below, &array; supplants the exact return types
-     because they are complicated and rarely need to be explicitly
-     written down.</para>
- 
-     <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	   orient="port" pgwide="0">
-      <title>Operators on &array;</title>
- 
-      <tgroup cols="2" align="left">
-       <thead>
-        <row>
- 	<entry>Operator</entry>
- 	<entry>Value</entry>
-        </row>
-       </thead>
-       <tbody>
-        <row>
- 	<entry>
-           <funcsynopsis>
- 	  <funcprototype>
- 	   <funcdef>Array <function>acos</function></funcdef>
- 	   <paramdef>const <parameter>Array&lt;Dim,T,EngineTag&gt;& a</parameter></paramdef>
- 	  </funcprototype>
- 	 </funcsynopsis>
-          </entry>
- 	<entry><para>an array containing the element-wise inverse
-          cosine of the array <parameter>a</parameter></para></entry>
-        </row>
-        <row>
- 	<entry>ADD ALL other operators appearing in <filename
- 							      class="headerfile">src/Array/ArrayOperators.h</filename>,
-         <filename
- 		  class="headerfile">src/Array/ArrayOperatorSpecializations.h</filename>,
-         <filename
- 		  class="headerfile">src/Array/PoomaArrayOperators.h</filename>,
-         and <filename
- 		      class="headerfile">src/Array/VectorArrayOperators.h</filename>.</entry>
-        </row>
-       </tbody>
-      </tgroup>
-     </table>
- 
-     <para>FINISH: Write one or two examples or refer to ones
-     previously in the text.</para>
-    </section>
- 
- 
-    <section id="arrays_ref-reductions">
-     <title>Reducing All &array; Elements to One Value</title>
- 
-     <para>These reduction functions repeatedly apply a binary
-     operation to all array elements to yield a value.  These functions
-     are similar to the Standard Template Library's
-     <function>accumulate</function> function.  For example,
-     <function>sum</function> repeatedly applies the binary plus
-     operator to all array elements, yielding the sum of all array
-     elements.</para>
- 
-     <para>FINISH: What order of operation, if any, is
-     guaranteed?</para>
- 
-     <para>FINISH: Add a table of the functions in <filename
- 							    class="headerfile">src/Array/Reductions.h</filename>.</para>
- 
-     <para>How does one use one's own binary function?  See <filename
-     class="headerfile">src/Engine/Reduction.h</filename>.</para>
-    </section>
- 
- 
-    <section id="arrays_ref-utilities">
-     <title>Utility Functions</title>
- 
-     <section id="arrays_ref-utilities-compression">
-      <title>Compressed Data</title>
- 
-      <para>Add a table containing
-      <function>elementsCompressed</function>,
-      <function>compressed</function>, <function>compress</function>,
-      and <function>uncompress</function>.</para>
-     </section>
- 
- 
-     <section id="arrays_ref-utilities-sizes">
-      <title>Centering Sizes and Number of Materials</title>
- 
-      <para>ADD: a description of <function>numMaterials</function> and
-      <function>centeringSize</function> found in <filename
-      class="headerfile">src/Field/Field.h</filename>.  These functions
-      are meaningless for &array; but are provided for consistency with
-      &field;.</para>
-     </section>
- 
-     <section id="arrays_ref-utilities-subfield">
-      <title>Obtaining Subfields</title>
- 
-      <para>ADD: a description of <function>subField</function> found
-      in <filename class="headerfile">src/Field/Field.h</filename>.
-      This function, meaningless for &array;, is provided for
-      consistency with &field;.</para>
-     </section>
-    </section>
- 
- 
-    <section id="arrays_ref-tmp">
-     <title>TMP: What do we do with these &hellip;? Remove this
-     section.</title>
- 
-     <para>QUESTION: Do we describe the &leaffunctor;s specialized for
-     &array;s in <filename
-     class="headerfile">src/Array/Array.h</filename> or in the &pete;
-     reference section?  What about the functions in <filename
-     class="headerfile">src/Array/CreateLeaf.h</filename>?</para>
- 
-     <para>QUESTION: What is an <type>EngineFunctor</type>?  We
-     probably should describe it in an analogous way as for
-     &leaffunctor;s.</para>
- 
-     <para>QUESTION: Where do we write about
-     <type>ExpressionTraits</type> for &array;s?</para>
- 
-     <para>QUESTION: Do we describe the <type>ElementProperties</type>
-     specialization at this place or in its section?</para>
- 
-     <para>QUESTION: Do we describe the <type>Patch</type>
-     specialization for &array;s (<filename
-     class="headerfile">src/Array/Array.h</filename>:1300) in this
-     place or in a section for patches?</para>
-    </section>
-   </chapter>
- 
- 
-   <chapter id="fields_ref">
-    <title>&field;s</title>
- 
-    <para>An &array; is a set of values indexed by
-    coordinates, one value per coordinate.  It models the computer
-    science idea of an array.  Similarly, a &field; is a set of values
-    indexed by coordinate.  It models the mathematical and physical
-    idea of a field represented by a grid of rectangular cells, each
-    having at least one value.  A &field;'s functionality is a superset
-    of an &array;'s functionality because:
-     <itemizedlist>
-      <listitem>
-       <para>A &field; is distributed through space so one can compute
-       the distances between cells.</para>
-      </listitem>
-      <listitem>
-       <para>Each cell can hold multiple values.  For example, a
-       rectangular cell can have one value on each of its faces.</para>
-      </listitem>
-      <listitem>
-       <para>Multiple materials can share the same cell.  For example,
-       different values can be stored in the same cell for carbon,
-       oxygen, and nitrogen.</para>
-      </listitem>
-     </itemizedlist>
-    Also, &field;s' values can be related by relations.  Thus, if one
-    field's values change, a dependent field's values can be
-    automatically computed when needed.  FIXME: See also the unfinished
-    works chapter's entry concerning relations and arrays.</para>
- 
-    <para>QUESTION: Should we add a picture comparing and contrasting
-    an array and a field?</para>
- 
-    <para>QUESTION: How much structure can be copied from the &array;
-    chapter?</para>
- 
-    <para>QUESTION: Where is <type>NewMeshTag</type>, defined in
-    <filename class="headerfile">src/Field/Field.h</filename>,
-    used?</para>
- 
-    <para>QUESTION: Do we describe the &leaffunctor;s specialized for
-    &field;s in <filename
-    class="headerfile">src/Field/Field.h</filename> or in the &pete;
-    reference section?  Use the same decision for &array;s.</para>
- 
-    <para>QUESTION: What do the structure and functions in <filename
-    class="headerfile">src/Field/Mesh/PositionFunctions.h</filename>
-    do?</para>
- 
- 
-    <section id="fields_ref-field">
-     <title>The &field; Container</title>
- 
-     <para>ADD: table of template parameters and table of compile-time
-     types and values.</para>
- 
- 
-     <section id="fields_ref-field-constructors">
-      <title>Constructors and Destructors</title>
- 
-      <para>ADD: this section similar to &array;s's constructor and
-      destructor section.</para>
-     </section>
- 
-     <section id="fields_ref-field-initializers">
-      <title>Initializers</title>
- 
-      <para>Add a table.</para>
-     </section>
- 
- 
-     <section id="fields_ref-field-access">
-      <title>Element Access</title>
- 
-      <para>ADD: a table ala &array;.  Be sure to include
-      <methodname>all</methodname>.</para>
-     </section>
- 
- 
-     <section id="fields_ref-field-component">
-      <title>Component Access</title>
- 
-      <para>ADD: a table ala &array;.</para>
-     </section>
- 
- 
-     <section id="fields_ref-field-subfields">
-      <title>Obtaining Subfields</title>
- 
-      <para>ADD: discussion and a table listing ways to obtain
-       subfields.  Although the implementation may treat subfield views
-       and other field views similarly (?Is this true?), they are
-       conceptually different ideas so we present them
-       separately.</para>
- 
-      <para>See <filename
- 			 class="headerfile">src/Field/Field.h</filename>'s
-       <methodname>operator[]</methodname>,
-       <methodname>subField</methodname>, &hellip;,
-       <methodname>material</methodname>.</para>
-     </section>
- 
- 
-     <section id="fields_ref-field-relations">
-      <title>Supporting Relations</title>
- 
-      <para>ADD: a table with the member functions including
-      <methodname>addRelation</methodname>,
-      <methodname>removeRelations</methodname>,
-      <methodname>applyRelations</methodname>, and
-      <methodname>setDirty</methodname>.</para>
-     </section>
- 
- 
-     <section id="fields_ref-field-accessors">
-      <title>Accessors</title>
- 
-      <para>ADD: a table using lines like <filename
-      class="headerfile">src/Field/Field.h</filename>:1243&ndash;1333.</para>
-     </section>
- 
- 
-     <section id="fields_ref-field-utilities">
-      <title>Utility Methods</title>
- 
-      <para>ADD: a table including
-       <methodname>makeOwnCopy</methodname>.</para>
-     </section>
- 
- 
-     <section id="fields_ref-field-implementation">
-      <title>Implementation Details</title>
- 
-      <para>ADD: a table similar to &array;'s.</para>
- 
-     </section>
- 
-    </section>
- 
- 
-    <section id="engines_ref-views">
-     <title>Views of &field;s</title>
- 
-     <para>Be sure to relate to &array; views.  Note only three
-     dimensions are supported.</para>
- 
-     <para>Be sure to describe <statement>f[i]</statement>.  Does this
-     refer to a particular material or a particular value within a
-     cell?  I do not remember.  See <type>SubFieldView</type> in
-     <filename class="headerfile">src/Field/Field.h</filename>.</para>
-    </section>
- 
- 
-    <section id="fields_ref-assignment">
-     <title>&field; Assignments</title>
- 
-     <para>ADD: Describe supported assignments, relating to &array;'s
-     assignments.</para>
- 
-     <para>UNFINISHED: Add a table containing assignment operators
-     found on <filename
- 		       class="headerfile">src/Field/Field.h</filename>:2097&ndash;2202
-     and 1512&ndash;1611.</para>
-    </section>
- 
- 
-    <section id="fields_ref-print">
-     <title>Printing &field;s</title>
- 
-     <para>QUESTION: How similar is this to printing &array;s?</para>
- 
-     <para>&field;s support output to but not input from IO streams.
-     In particular, output to <type>ostream</type>s and file streams is
-     supported.</para>
- 
-     <para>Add a table, using <filename
- 				       class="headerfile">src/Field/Field.h</filename>, lines
-     1996&ndash;2009.  See the implementation in <filename
- 							  class="headerfile">src/Field/PrintField.h</filename>.</para>
-    </section>
- 
- 
-    <section id="fields_ref-reductions">
-     <title>Combining &field; Elements</title>
- 
-     <para>Like &array;s, &field;s support reduction of all elements to
-     one value.  Additionally, the latter supports computing a field's
-     values using field stencils.  QUESTION: How do I describe this
-     with a minimum of jargon?</para>
- 
-     <para>ADD: something similar to &array; reductions.</para>
- 
-     <para>FINISH: Add a table of the functions in <filename
-     class="headerfile">src/Field/FieldReductions.h</filename>.</para>
- 
-     <para>FINISH: Add a table of the functions in <filename
-     class="headerfile">src/Field/DiffOps/FieldOffsetReductions.h</filename>.
-     QUESTION: Why is only <function>sum</function> defined?</para>
-    </section>
- 
- 
-    <section id="fields_ref-expressions">
-     <title>Expressions Involving &field;s</title>
- 
-     <para>Do something similar to &array;'s section.  See the
-     operations defined in <filename
-     class="headerfile">src/Field/FieldOperators.h</filename>,
-     <filename
-     class="headerfile">src/Field/FieldOperatorSpecializations.h</filename>,
-     <filename
-     class="headerfile">src/Field/PoomaFieldOperators.h</filename>, and
-     <filename
-     class="headerfile">src/Field/VectorFieldOperators.h</filename>.</para>
- 
-     <para>Some operations involve both &array; and &field;
-     parameters.  Where do we list them?</para>
-    </section>
- 
- 
-    <section id="fields_ref-stencils">
-     <title>&field; Stencils: Faster, Local Computations</title>
- 
-     <para>ADD: a description of a stencil.  Why is it needed?  How
-     does a user use it?  How does a user know when to use one?  Add
-     documentation of the material from <filename
-     class="headerfile">src/Field/DiffOps/FieldStencil.h</filename>.</para>
- 
-     <para>How is <type>FieldShiftEngine</type> used by &field;
-     stencils?  Should it be described here or in the &engine; section?
-     See the the code in <filename
-     class="headerfile">src/Field/DiffOps/FieldShiftEngine.h</filename>.</para>
-    </section>
- 
- 
-    <section>
-     <title>Cell Volumes, Face Areas, Edge Lengths, Normals</title>
- 
-     <para>ADD: a description of these functions.  See <filename
-     class="headerfile">src/Field/Mesh/MeshFunctions.h</filename>.
-     These are initialized in, e.g., <filename
-     class="headerfile">src/Field/Mesh/UniformRectilinearMesh.h</filename>.
-     Note that these do not work for <type>NoMesh</type>.</para>
-    </section>
- 
- 
-    <section id="fields_ref-div">
-     <title>Divergence Operators</title>
- 
-     <para>ADD: a table having divergence operators, explaining the
-     current restrictions imposed by what is implemented.  See
-     <filename class="headerfile">src/Field/DiffOps/Div.h</filename>
-     and <filename
-     class="headerfile">src/Field/DiffOps/Div.UR.h</filename>.  What
-     restrictions does <quote>UR</quote> (mesh) connote?</para>
-    </section>
- 
- 
-    <section id="fields_ref-utilities">
-     <title>Utility Functions</title>
- 
-     <section id="fields_ref-utilities-compression">
-      <title>Compressed Data</title>
- 
-      <para>Add a table containing
-      <function>elementsCompressed</function>,
-      <function>compressed</function>, <function>compress</function>,
-      and <function>uncompress</function>.</para>
-     </section>
- 
- 
-     <section id="fields_ref-utilities-sizes">
-      <title>Centering Sizes and Number of Materials</title>
- 
-      <para>ADD: a description of <function>numMaterials</function> and
-      <function>centeringSize</function> found in <filename
-      class="headerfile">src/Field/Field.h</filename>.</para>
- 
-      <para>QUESTION: How do these relate to any method functions?</para>
-     </section>
- 
- 
-     <section id="fields_ref-utilities-subfield">
-      <title>Obtaining Subfields</title>
- 
-      <para>ADD: a description of <function>subField</function> found
-      in <filename
-      class="headerfile">src/Field/Field.h</filename>.</para>
-     </section>
- 
-    </section>
- 
- 
-    <section id="fields_ref-centerings">
-     <title>&field; Centerings</title>
- 
-     <para>DO: Describe the purpose of a centering and its definition.
-     Describe the ability to obtain canonical centerings.  Explain how
-     to construct a unique centering.  See <filename
-     class="headerfile">src/Field/FieldDentering.h</filename>.</para>
-    </section>
- 
- 
-    <section id="fields_ref-fieldoffsets">
-     <title>Relative &field; Positions</title>
- 
-     <para>Permit specifying field positions relative to a field
-     location.  Describe <type>FieldOffset</type> and
-     <type>FieldOffsetList</type>.  See <filename
-     class="headerfile">src/Field/FieldOffset.h</filename></para>
-    </section>
- 
- 
-    <section id="fields_ref-nearestneighbors">
-     <title>Computing Close-by Field Positions</title>
- 
-     <para>Given a field location, return the set of field locations
-     that are closest using ?Manhattan? distance.  See <filename
-     class="headerfile">src/Field/NearestNeighbors.h</filename>.</para>
-    </section>
- 
- 
-    <section id="fields_ref-mesh">
-     <title>Mesh ???</title>
- 
-     <para>Unlike &array;s, &field;s are distributed throughout space
-     so distances between values within the &field can be computed.  A
-     &field;'s mesh stores this spatial distribution.</para>
- 
-     <para>QUESTION: What do we need to write about meshes?  What is
-     unimportant implementation and what should be described in this
-     reference section?</para>
- 
-     <para>QUESTION: Where in here should emphasize vertex, not cell,
-     positions?  <quote>VERTEX</quote> appears repeatedly in <filename
-     class="headerfile">src/Field/Mesh/NoMesh.h</filename>.</para>
- 
-     <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	   orient="port" pgwide="0">
-      <title>Mesh Types</title>
- 
-      <tgroup cols="2" align="left">
-       <thead>
-        <row>
- 	<entry>Mesh Type</entry>
- 	<entry>Description</entry>
-        </row>
-       </thead>
-       <tbody>
-        <row>
- 	<entry><type>NoMesh&lt;Dim&gt;</type></entry>
- 	<entry><para>no physical spacing, causing a &field; to mimic
- 	an &array; with multiple engines.</para></entry>
-        </row>
-        <row>
- 	<entry><type>UniformRectilinearMesh&lt;Dim,T&gt;</type></entry>
- 	<entry><para>physical spacing formed by the Cartesian product
- 	of ????.</para></entry>
-        </row>
-       </tbody>
-      </tgroup>
-     </table>
- 
- 
-     <section id="fields_ref-mesh-accessors">
-      <title>Mesh Accessors</title>
- 
-      <para>ADD: a table listing accessors, explaining the difference
-      between (physical and total) and (cell and vertex) domains.  See
-      <filename class="headerfile">src/Field/Mesh/NoMesh.h</filename>.
-      Also, include <function>spacings</function> and
-      <function>origin</function> in <filename
-      class="headerfile">src/Field/Mesh/UniformRectilinearMesh.h</filename>.
-      Note <type>NoMesh</type> does not provide the latter two.</para>
-     </section>
- 
-    </section>
- 
- 
-    <section id="fields_ref-tmp">
-     <title>TMP: What do we do with these &hellip;? Remove this
-     section.</title>
- 
-     <para>QUESTION: Do we describe the <type>Patch</type>
-     specialization for &field; at this place or in some common place?
-     Follow &array;'s lead.</para>
- 
-     <para>QUESTION: Where do we describe <type>CreateLeaf</type> and
-     <type>MakeFieldReturn</type> in <filename
-     class="headerfile">src/Field/FieldCreateLeaf.h</filename> and
-     <filename
-     class="headerfile">src/Field/FieldMakeReturn.h</filename>.</para>
- 
-     <para>QUESTION: What do we do with <type>FieldEnginePatch</type>
-     in <filename
-     class="headerfile">src/Field/FieldEngine/FieldEnginePatch.h</filename>.</para>
-    </section>
-   </chapter>
- 
- 
-   <chapter id="engines_ref">
-    <title>&engine;s</title>
- 
-    <para>From a user's point of view, a container makes data available
-    for reading and writing.  In fact, the container's &engine; stores
-    the data or, if the data is computed, performs a computation to
-    yield the data.</para>
- 
-    <para>FINISH: Introduce the various types of engines.  Add a table
-    with a short description of each engine type.</para>
- 
-    <para>FINISH: First, we specify a generic &engine;'s interface.
-    Then, we present &engine; specializations.</para>
-    
-    <table frame="none" colsep="0" rowsep="0" tocentry="1"
- 	  orient="port" pgwide="0">
-     <title>Types of &engine;s</title>
- 
-     <tgroup cols="3" align="left">
-      <thead>
-       <row>
-        <entry>Engine Type</entry>
-        <entry>Engine Tag</entry>
-        <entry>Description</entry>
-       </row>
-      </thead>
-      <tbody>
-       <row>
-        <entry><para>Brick</para></entry>
-        <entry><para><literal>Brick</literal></para></entry>
-        <entry><para>Explicitly store all elements in, e.g., a &cc;
- 	array.</para></entry>
-       </row>
-       <row>
-        <entry><para>Compressible</para></entry>
-        <entry><para><literal>CompressibleBrick</literal></para></entry>
-        <entry><para>If all values are the same, use constant storage
- 	for that single value.  Otherwise, explicitly store all
- 	elements.</para></entry>
-       </row>
-       <row>
-        <entry><para>Constant</para></entry>
-        <entry><para><literal>ConstantFunction</literal></para></entry>
-        <entry><para>Returns the same constant value for all
-        indices.</para></entry>
-       </row>
-       <row>
-        <entry><para>Dynamic</para></entry>
-        <entry><para><literal>Dynamic</literal></para></entry>
-        <entry><para>Manages a contiguous, local, one-dimensional,
-        dynamically resizable block of data.</para></entry>
-       </row>
-       <row>
-        <entry><para>Component Forwarding</para></entry>
-        <entry><para><literal>CompFwd&lt;EngineTag,
-        Components&gt;</literal></para></entry>
-        <entry><para>Returns the specified components from
-        <literal>EngineTag</literal>'s engine.  Components are
-        <quote>pieces</quote> of multi-value elements such as vectors
-        and tensors.</para></entry>
-       </row>
-       <row>
-        <entry><para>Expression</para></entry>
-        <entry><para><literal>ExpressionTag&lt;Expr&gt;</literal></para></entry>
-        <entry><para>Returns the value of the specified &pete;
-        expression.</para></entry>
-       </row>
-       <row>
-        <entry><para>Index Function</para></entry>
-        <entry><para><literal>IndexFunction&lt;Functor&gt;</literal></para></entry>
-        <entry><para>Makes the function
-        <literal>Functor</literal>accepting indices mimic an
-        array.</para></entry>
-       </row>
-       <row>
-        <entry><para>MultiPatch</para></entry>
-        <entry><para><literal>MultiPatch&lt;LayoutTag,PatchTag&gt;</literal></para></entry>
-        <entry><para>Support distributed computation using several
-        processors (???contexts???).  <literal>LayoutTag</literal>
-        indicates how the entire array is distributed among the
-        processors.  Each processor uses a <literal>PatchTag</literal>
-        engine.</para></entry>
-       </row>
-       <row>
-        <entry><para>Remote</para></entry>
-        <entry><para><literal>Remote&lt;EngineTag&gt;</literal></para></entry>
-        <entry><para>unknown</para></entry>
-       </row>
-       <row>
-        <entry><para>Remote Dynamic</para></entry>
-        <entry><para><literal>Remote&lt;Dynamic&gt;</literal></para></entry>
-        <entry><para>unknown: specialization</para></entry>
-       </row>
-       <row>
-        <entry><para>Stencil</para></entry>
-        <entry><para><literal>StencilEngine&lt;Function,
-        Expression&gt;</literal></para></entry>
-        <entry><para>Returns values computed by applying the
-        user-specified function to sets of contiguous values in the
-        given engine or container.  Compare with user function
-        engines.</para></entry>
-       </row>
-       <row>
-        <entry><para>User Function</para></entry>
-        <entry><para><literal>UserFunctionEngine&lt;UserFunction,Expression&gt;</literal></para></entry>
-        <entry><para> Returns values computed by applying the
-        user-specified function to the given engine or container.
-        QUESTION: Is the following claim correct? For each returned
-        value, only one value from the engine or container is
-        used.</para></entry>
-       </row>
-      </tbody>
-     </tgroup>
-    </table>
- 
-    <para>QUESTION: Where do we describe views?</para>
- 
-    <para>QUESTION: What does <type>NewEngine</type> do?  Should it be
-     described when describing views?  Should it be omitted as an
-     implementation detail?</para>
- 
-    <para>QUESTION: Where do we describe &engine; patches found in
-    <filename class="headerfile">src/Engine/EnginePatch.h</filename>?
-    All patch data in a separate chapter or engine-specific pieces in
-    this chapter?</para>
- 
-    <para>QUESTION: What is <function>notifyEngineWrite</function>?
-    See also <filename
- 		      class="headerfile">src/Engine/NotifyEngineWrite.h</filename>.</para>
- 
-    <para>QUESTION: What aspect of MultiPatch uses <type>IsValid</type> in
-    <filename
- 	     class="headerfile">src/Engine/IsValidLocation.h</filename>?</para>
- 
-    <para>QUESTION: Who uses intersections?  Where should this be
-    described?  See <filename
- 			     class="headerfile">src/Engine/Intersector.h</filename>, <filename
- 											       class="headerfile">src/Engine/IntersectEngine.h</filename>, and
-    <filename
- 	     class="headerfile">src/Engine/ViewEngine.h</filename>.</para>
- 
-    <section id="engines_ref-compile_time_interface">
-     <title>&engine; Compile-Time Interface</title>
- 
-     <para>ADD: a table of template parameters ala &array;.  ADD:
-     compile-time types and values.</para>
-    </section>
- 
- 
-    <section id="engines_ref-constructors">
-     <title>Constructors and Destructors</title>
- 
-     <para>ADD: a table of constructors and destructors ala
-     &array;'s.</para>
-    </section>
- 
- 
-    <section id="engines_ref-access">
-     <title>Element Access</title>
- 
-     <para>ADD: a table with <methodname>read</methodname> and
-     <methodname>operator()</methodname>.</para>
-    </section>
- 
- 
-    <section id="engines_ref-accessors">
-     <title>Accessors</title>
- 
-     <para>ADD: a table of accessors.</para>
-    </section>
- 
- 
-    <section id="engines_ref-assignments">
-     <title>&engine; Assignments</title>
- 
-     <para>similar to &array;'s assignments.  shallow copies.  ADD: a
-     table with one entry</para>
-    </section>
- 
- 
-    <section id="engines_ref-utilities">
-     <title>Utility Methods</title>
- 
-     <para>ADD: a table including
-     <methodname>makeOwnCopy</methodname>.</para>
- 
-     <para>QUESTION: What are <methodname>dataObject</methodname>,
-     <methodname>isShared</methodname>, and related methods?</para>
-    </section>
- 
- 
-    <section id="engines_ref-implementation">
-     <title>Implementation Details</title>
- 
-     <para>ADD: this section.  Explain that
-     <varname>dataBlock_m</varname> and <varname>data_m</varname> point
-     to the same place.  The latter speeds access, but what is the
-     purpose of the former?</para>
-    </section>
- 
- 
-    <section id="engines_ref-brick">
-     <title>Brick and BrickView Engines</title>
- 
-     <para>ADD: description of what a brick means.  ADD: whatever
-     specializations the class has, e.g.,
-     <methodname>offset</methodname>.</para>
- 
-     <para>QUESTION: What does <type>DoubleSliceHelper</type> do?</para>
-    </section>
- 
- 
-    <section id="engines_ref-compressible">
-     <title>Compressible Brick and BrickView Engines</title>
- 
-     <para>ADD this.</para>
-    </section>
- 
- 
-    <section id="engines_ref-dynamic">
-     <title>Dynamic and DynamicView Engines:</title>
- 
-     <para>ADD this.  Manages a contiguous, local, resizable, 1D block
-     of data.</para>
-    </section>
- 
- 
-    <section id="engines_ref-components">
-     <title>Component Engines</title>
- 
-     <para>I believe these implement array component-forwarding.  See
-     <filename
- 	      class="headerfile">src/Engine/ForwardingEngine.h</filename>.</para>
-    </section>
- 
- 
-    <section id="engines_ref-expressions">
-     <title>Expression Engines</title>
- 
-     <para>Should this be described in the &pete; section?  Unlikely.
-     See <filename
- 		  class="headerfile">src/Engine/ExpressionEngine.h</filename>.</para>
-    </section>
- 
- 
-    <section id="engines_ref-enginefunctor">
-     <title>&engine; Functors</title>
- 
-     <para>QUESTION: What is an <type>EngineFunctor</type>?  Should it
-     have its own section?  See <filename
- 					 class="headerfile">src/Engine/EngineFunctor.h</filename>.</para>
-    </section>
- 
- 
-    <section id="engines_ref-fieldengines">
-     <title><type>FieldEngine</type>: A Hierarchy of &engine;s</title>
- 
-     <para>A &field; consists of a hierarchy of materials and
-     centerings.  These are implemented using a hierarchy of engines.
-     See <filename
- 		  class="headerfile">src/Field/FieldEngine/FieldEngine.h</filename>
-     and <filename
- 		  class="headerfile">src/Field/FieldEngine/FieldEngine.ExprEngine.h</filename>.</para>
-    </section>
-   </chapter>
- 
- 
-   <chapter id="benchmarks_ref">
-    <title>&benchmark; Programs</title>
- 
-    <para>Explain how to use &benchmark; programs, especially the
-    options.  Explain how to write a &benchmark; program.  See also
-    <filename class="headerfile">src/Utilities/Benchmark.h</filename>
-    and <filename
-    class="libraryfile">src/Utilities/Benchmark.cmpl.cpp</filename>.</para>
- 
-   </chapter>
- 
- 
-   <chapter id="layouts_ref">
-    <title>Layouts and Partitions: Distribute Computation Among
-    Contexts</title>
- 
-    <para>QUESTION: What is the difference between
-    <type>ReplicatedTag</type> and <type>DistributedTag</type>?</para>
- 
-   </chapter>
- 
- 
-   <chapter id="pete_ref">
-    <title>&pete;: Evaluating Parallel Expressions</title>
- 
-    <section>
-     <title>UNKNOWN</title>
- 
-     <section id="pete_ref-unknown-leaf-tags">
-      <title>Leaf Tag Classes</title>
- 
-      <para><type>NotifyPreReadTag</type> indicates a term is about to
-      be read.  Why is this needed?  Defined in <filename
-      class="headerfile">src/Utilities/NotifyPreRead.h</filename>.</para>
-     </section>
-    </section>
- 
-   </chapter>
- 
- 
-   <chapter id="views_ref">
-    <title>Views</title>
- 
-    <para>QUESTION: Should this have its own chapter or be part of a
-    container chapter?</para>
- 
-    <para>Describe <type>View0</type>, <type>View1</type>, &hellip;,
-    <type>View7</type> and <type>View1Implementation</type>.</para>
- 
-    <para>QUESTION: What causes the need for <type>AltView0</type> and
-    <type>AltComponentView</type>?</para>
- 
-    <para>Be sure to describe <type>ComponentView</type> in the same
-    place.  This is specialized for &array;s in <filename
-    class="headerfile">src/Array/Array.h</filename>:1323&ndash;1382.</para>
- 
-    <section>
-     <title><type>ViewIndexer&lt;Dim,Dim2&gt;</type></title>
- 
-     <para>Defined in <filename
-     class="headerfile">src/Utilities/ViewIndexer.h</filename>, this
-     type translates indices between a domain and a view of it.</para>
-    </section>
-   </chapter>
- 
-   <chapter id="threads_ref">
-    <title>Threads</title>
- 
-    <para>Perhaps include information in <filename
-    class="headerfile">src/Engine/DataObject.h</filename>.</para>
- 
-    <para>&pooma; options include UNFINISHED</para>
- 
-   </chapter>
- 
- 
-   <chapter id="utilities_ref">
-    <title>Utility Types</title>
- 
-    <para>TMP: What is a good order?</para>
- 
-    <section id="utilities_ref-options">
-     <title><type>Options</type>: Varying Run-Time Execution</title>
- 
-     <para>Each &pooma; executable has a <type>Options</type> object,
-     created by <function>Pooma::initialize</function>, storing
-     run-time configurable values found in <varname>argv</varname>.
-     Default options are found in
-     <methodname>Options::usage</methodname>.</para>
- 
-     <para>See <filename
-     class="headerfile">src/Utilities/Options.h</filename> and
-     <filename
-     class="libraryfile">src/Utilities/Options.cmpl.cpp</filename>.</para>
- 
-     <para>Scatter the specific options to other parts of the
-     manual.</para>
-    </section>
- 
-    <section id="utilities_ref-assertions">
-     <title>Check Correctness: <type>CTAssert</type>,
-     <type>PAssert</type>, <type>PInsist</type>,
-     <type>SameType</type></title>
- 
-     <para>Assertions ensure program invariants are obeyed.
-     <type>CTAssert</type>, checked at compile time, incur no run-time
-     cost.  <type>PAssert</type> and <type>PInsist</type> are checked
-     to run-time, the latter producing an explanatory message if the
-     assertion fails.  Compiling with <envar>NOCTAssert</envar> and
-     <envar>NOPTAssert</envar> disable these checks.  Compiling with just
-     <envar>NOPTAssert</envar> disables only the run-time checks.</para>
- 
-     <para><type>SameType</type> ensures, at compile-time, two types
-     are the same.</para>
- 
-     <para>These are implemented in <filename
-     class="headerfile">src/Utilities/PAssert.h</filename> and
-     <filename
-     class="libraryfile">src/Utilities/PAssert.cmpl.cpp</filename>.</para>
-    </section>
- 
-    <section id="utilities_ref-clock">
-     <title><type>Clock</type>: Measuring a Program's Execution Time</title>
- 
-     <para>See <filename
-     class="headerfile">src/Utilities/Clock.h</filename>.</para>
-    </section>
- 
- 
-    <section id="utilities_ref-smart_pointers">
-     <title>Smart Pointers: <type>RefCountedPtr</type>,
-     <type>RefCountedBlockPtr</type>, and
-     <type>DataBlockPtr</type></title>
- 
-     <para>See <filename
-     class="headerfile">src/Utilities/{RefCountedPtr,RefCountedBlockPtr,DataBlockPtr}.h</filename>.
-     <filename class="headerfile">src/Utilities/RefCounted.h</filename>
-     helps implement it.  <type>DataBlockPtr</type> uses
-     &smarts;.</para>
-    </section>
- 
-    <section id="utilities_ref-inform">
-     <title><type>Inform</type>: Formatted Output for Multi-context
-     Execution</title>
- 
-     <para>See <filename
-     class="headerfile">src/Utilities/Inform.h</filename> and <filename
-     class="libraryfile">src/Utilities/Inform.cmpl.cpp</filename>.</para>
-    </section>
- 
-    <section id="utilities_ref-statistics">
-     <title><type>Statistics</type>: Report &pooma; Execution Statistics</title>
- 
-     <para>Collect and print execution statistics.  Defined in
-     <filename
-     class="headerfile">src/Utilities/Statistics.h</filename>.</para>
-    </section>
- 
-    <section id="utilities_ref-random_numbers">
-     <title>Random Numbers: <type>Unique</type></title>
- 
-     <para>See <filename
-     class="headerfile">src/Utilities/Unique.h</filename>.</para>
-    </section>
-   </chapter>
- 
- 
-   <chapter id="implementation_types_ref">
-    <title>Types for Implementing &pooma;</title>
- 
-    <para>TMP: What is a good order?</para>
- 
-    <para>Describe types defined to implement &pooma; but that users do
-    not directly use.  This chapter has lower priority than other
-    chapters since users (hopefully) do not need to know about these
-    classes.</para>
- 
-    <section id="implementation_types_ref-tester">
-     <title><type>Tester</type>: Check Implementation Correctness</title>
- 
-     <para>&pooma; implementation test programs frequently consist of a
-     series of operations followed by correctness checks.  The
-     <type>Tester</type> object supports these tests, returning a
-     boolean whether all the correctness checks yield true.  Under
-     verbose output, messages are printed for each test.  See <filename
-     class="headerfile">src/Utilities/Tester.h</filename>.</para>
-    </section>
- 
-    <section id="implementation_types_ref-elementproperties">
-     <title><type>ElementProperties&lt;T&gt;</type>: Properties a Type
-     Supports</title>
- 
-     <para>This traits class permits optimizations in other templated
-     classes.  See <filename
-     class="headerfile">src/Utilities/ElementProperties.h</filename>.</para>
- 
-    </section>
- 
-    <section id="implementation_types_ref-typeinfo">
-     <title><type>TypeInfo&lt;T&gt;</type>: Print a String Describing
-     the Type</title>
- 
-     <para>Print a string describing the type.  Defined in <filename
-     class="headerfile">src/Utilities/TypeInfo.h</filename>.  It is
-     specialized for other types in other files, e.g., <filename
-     class="headerfile">src/Engine/EngineTypeInfo.h</filename> and
-     <filename class="headerfile">src/Field/FieldTypeInfo.h</filename>.
-     Is this a compile-time version of RTTI?</para>
-    </section>
- 
-    <section id="implementation_types_ref-looputils">
-     <title><type>LoopUtils</type>: Loop Computations at Compile Time</title>
- 
-     <para>At compile time, <type>LoopUtils</type> supports copying
-     between arrays and computing the dot product of arrays.  See
-     <filename
- 	      class="headerfile">src/Utilities/MetaProg.h</filename>.</para>
-    </section>
- 
-    <section id="implementation_types_ref-modelelement">
-     <title><type>ModelElement&lt;T&gt;</type>: Wrap a Type</title>
- 
-     <para>A wrapper class used to differentiate overloaded functions.
-     Defined in <filename
-     class="headerfile">src/Utilities/ModelElement.h</filename>.  Used
-     only by &array; and <type>DynamicArray</type>.</para>
-    </section>
- 
-    <section id="implementation_types_ref-wrappedint">
-     <title><type>WrappedInt&lt;int&gt;</type>: Wrap a Number</title>
- 
-     <para>A wrapper class used to differentiate overloaded functions
-     among different integers.  Defined in <filename
-     class="headerfile">src/Utilities/WrappedInt.h</filename>.  Is this
-     class deprecated?  Is it even necessary?</para>
-    </section>
-    
-    <section id="implementation_types_ref-empty_classes">
-     <title>Supporting Empty Classes</title>
- 
-     <para>The <type>NoInit</type> tag class indicates certain
-     initializations should be skipped.  Defined in <filename
-     class="headerfile">src/Utilities/NoInit.h</filename>.</para>
- 
-     <para>FIXME: Should be macro, not function.
-     <function>POOMA_PURIFY_CONSTRUCTORS</function> generates an empty
-     constructor, copy constructor, and destructor to avoid &purify;
-     warnings.  Defined in <filename
-     class="headerfile">src/Utilities/PurifyConstructors.h</filename>.</para>
- 
-    </section>
- 
-    <section id="implementation_types_ref-pool">
-     <title><type>Pooled&lt;T&gt;</type>: Fast Memory Allocation of
-     Small Blocks</title>
- 
-     <para><type>Pooled&lt;T&gt;</type> speeds allocation and
-     deallocation of memory blocks for small objects with
-     type&nbsp;<type>T</type>.  Defined in <filename
-     class="headerfile">src/Utilities/Pooled.h</filename>, it is
-     implemented in <filename
-     class="headerfile">src/Utilities/Pool.h</filename> and <filename
-     class="libraryfile">src/Utilities/Pool.cmpl.cpp</filename>.
-     <filename class="headerfile">src/Utilities/StaticPool.h</filename>
-     no longer seems to be used.</para>
-    </section>
- 
-    <section id="implementation_types_ref-uninitialized_vector">
-     <title><type>UninitializedVector&lt;T,Dim&gt;</type>: Create
-     Without Initializing</title>
- 
-     <para>This class optimizes creation of an array of objects by
-     avoiding running the default constructors.  Later initialization
-     can occur, perhaps using a loop that can be unrolled.  Defined in
-     <filename
-     class="headerfile">src/Utilities/UninitializedVector.h</filename>,
-     this is used only by <type>DomainTraits</type>.</para>
-    </section>
-   </chapter>
- 
-   <chapter id="implementation_algorithms_ref">
-    <title>Algorithms for Implementing &pooma;</title>
- 
-    <para>In <filename>src/Utilities/algorithms.h</filename>,
-    <function>copy</function>, <function>delete_back</function>, and
-    <function>delete_shiftup</function> provide additional algorithms
-    using iterators.</para>
-   </chapter>
- 
- 
-   <chapter id="where-place-these_ref">
-    <title>TMP: Where do we describe these files?</title>
- 
-    <itemizedlist>
-     <listitem>
-      <para><filename
- 		     class="headerfile">src/Utilities/Conform.h</filename>: tag for
-      checking whether terms in expression have conforming
-      domains</para>
-     </listitem>
- 
-     <listitem>
-      <para><filename
- 		     class="headerfile">src/Utilities/DerefIterator.h</filename>:
-      <type>DerefIterator&lt;T&gt;</type> and
-      <type>ConstDerefIterator&lt;T&gt;</type> automatically
-      dereference themselves to maintain <literal>const</literal>
-      correctness.  <!-- FIXME: s/literal/keyword/ --></para>
-     </listitem>
- 
-     <listitem>
-      <para><filename
- 		     class="headerfile">src/Utilities/Observable.h</filename>,
-      <filename class="headerfile">src/Utilities/Observer.h</filename>,
-      and <filename
- 		   class="headerfile">src/Utilities/ObserverEvent.h</filename>:
-      <type>Observable&lt;T&gt;</type>,
-      <type>SingleObserveable&lt;T&gt;</type>,
-      <type>Observer&lt;T&gt;</type>, and <type>ObserverEvent</type>
-      implement the observer pattern.  What is the observer pattern?
-      Where is this used in the code?</para>
-     </listitem>
-    </itemizedlist>
- 
-   </chapter>
- 
-  </part>
- 
- 
-  <appendix id="future_development">
-   <title>Future Development</title>
- 
-   <section id="future_development-particles">
-    <title>Particles</title>
- 
-    <para><filename
- 		   class="libraryfile">docs/ParticlesDoc.txt</filename> has
-    out-of-date information.</para>
- 
-    <para>See Section&nbsp;3.2.3 of
-    <filename>papers/pooma.ps</filename> for an out-of-date
-    description.</para>
- 
-    <para><filename>papers/jvwr.ps</filename> concerns mainly
-    particles.  <filename>papers/8thSIAMPOOMAParticles.pdf</filename>,
-    by Julian Cummings and Bill Humphrey, concerns parallel particle
-    simulations.  <filename>papers/iscope98linac.pdf</filename>
-    describes a particle beam simulation using &pooma;; it mainly
-    concerns particles.</para>
- 
-    <section>
-     <title>Particles</title>
- 
-     <para>Do we want to include such a section?</para>
- 
-     <para>Section&nbsp;3, "Sample Applications" of
-     <filename>papers/SiamOO98_paper.ps</filename> describes porting a
-     particle program written using High-Performance Fortran to
-     &pooma; and presumably why particles were added to &pooma;.  It
-     also describes <application>MC++</application>, a Monte Carlo
-     neutron transport code.</para>
- 
-    </section>
- 
-   </section>
- 
- 
-   <section id="future_development-composition_engine">
-    <title>Composition of &engine;s</title>
- 
-    <para>The i,j-th element of the composition
-    <varname>a</varname>&compfn;<varname>b</varname> of two arrays
-    <varname>a</varname> and <varname>b</varname> equals a(b(i,j)).
-    The composition engine tagged <literal>IndirectionTag&lt;Array1,
-    Array2&gt;</literal>, defined in <filename
- 					      class="headerfile">src/Engine/IndirectionEngine.h</filename> is
-    unfinished.</para>
-   </section>
- 
- 
-   <section id="future_development-container_consistency">
-    <title>Improving Consistency of Container Interfaces</title>
- 
-    <section id="future_development-container_consistency-array_relations">
-     <title>Relations for &array;s</title>
- 
-     <para>Do &array;s currently support relations?  If not, why not?
-    Should they be added?</para>
-    </section>
- 
-    <section id="future_development-container_consistency-dimensions">
-     <title>Supporting the Same Number of Dimensions</title>
- 
-     <para>&array; and &field; should support the same maximum number
-     of dimensions.  Currently, &array;s support seven dimensions and
-     &field;s support only three.  By definition, &dynamicarray;
-     supports only one dimension.</para>
- 
-     <para>Relations for &array;s.</para>
- 
-     <para>External guards for &array;s.</para>
-    </section>
- 
-   </section>
- 
- 
-   <section id="future_development-where">
-    <title><function>where</function> Proxies</title>
- 
-    <para>QUESTION: Do we even discuss this broken
-    <quote>feature</quote>?  Where is it used?  Some related code is in
-    <filename>src/Array/Array.h</filename>:2511&ndash;2520.</para>
-   </section>
- 
- 
-   <section id="future_development-long_term">
-    <title>Very Long Term Development Ideas</title>
- 
-    <para>Describe how to write a new configuration file.</para>
-   </section>
- 
-  </appendix>
- 
- 
-  <appendix id="installation">
-   <title>Obtaining and Installing &pooma;</title>
- 
-   <para>ADD: Write this section, including extensive instructions
-   for Unix, MS Windows, and MacOS.  List the configuration options.
-   Be sure to describe configuring for parallel execution.</para>
- 
-   <section id="installation-distributed_computing">
-    <title>Supporting Distributed Computation</title>
- 
-    <para>To use multiple processors with &pooma; requires installing
-    the &cheetah; messaging library and an underlying messaging library
-    such as the Message Passing Interface (&mpi;) Communications
-    Library or the &mm; Shared Memory Library.  In this section, we
-    first describe how to install &mm;.  Read the section only if using
-    &mm;, not &mpi;.  Then we describe how to install &cheetah; and
-    configure &pooma; to use it.</para>
- 
-    <section id="installation-distributed_computing-mm">
-     <title>Obtaining and Installing the &mm; Shared Memory Library</title>
- 
-     <para>&cheetah;, and thus &pooma;, can use Ralf Engelschall's &mm;
-     Shared Memory Library to pass messages between processors.  For
-     example, the &author; uses this library on a two-processor
-     computer running &linux;.  The library, available at
-     http://www.engelschall.com/sw/mm/, is available for free and has
-     been successfully tested on a variety of Unix platforms.</para>
- 
-     <para>We describe how to download and install the &mm; library.
-      <orderedlist spacing="compact">
- 	<listitem>
- 	 <para>Download the library from the &pooma; Download page
-          available off the &pooma; home page (&poomaHomePage;).</para>
- 	</listitem>
- 	<listitem>
- 	 <para>Extract the source code using <command>tar xzvf
-          mm-1.1.3.tar.gz</command>.  Move into the resulting source
-          code directory <filename
-          class="directory">mm-1.1.3</filename>.</para>
- 	</listitem>
- 	<listitem>
- 	 <para>Prepare to compile the source code by configuring it
-          using the <command>configure</command> command.  To change
-          the default installation directory <filename
-          class="directory">/usr/local</filename>, specify
-          <command>--prefix=<replaceable>directory</replaceable></command>
-          option.  The other configuration options can be listed by
-          specifying the <command>--help</command> option.  Since the
-          &author; prefers to keep all &pooma;-related code in his
-          <filename class="directory">pooma</filename>subdirectory, he
-          uses <command>./configure
-          --prefix=${HOME}/pooma/mm-1.1.3</command>.</para>
- 	</listitem>
- 	<listitem>
- 	 <para>Create the library by issuing the <command>make</command>
-      command.  This compiles the source code using a &c; compiler.  To
-      use a different compiler than the &mm; configuration chooses, set
-      the <envar>CC</envar> to the compiler before configuring.</para>
-       </listitem>
-     <listitem>
-      <para>Optionally test the library by issuing the <command>make
-      test</command> command.  If successful, the penultimate line
-      should be <computeroutput>OK - ALL TESTS SUCCESSFULLY
-      PASSED</computeroutput>.</para>
-     </listitem>
-     <listitem>
-      <para>Install the &mm; Library by issuing the <command>make
-      install</command> command.  This copies the library files to the
-      installation directory.  The <filename
-      class="directory">mm-1.1.3</filename> directory containing the
-      source code may now be removed.</para>
-     </listitem>
-    </orderedlist>
-    </para>
-    </section>
- 
-    
-    <section id="installation-distributed_computing-cheetah">
-     <title>Obtaining and Installing the &cheetah; Messaging Library</title>
- 
-     <para>The &cheetah; Library decouples communication from
-     synchronization.  Using asynchronous messaging rather than
-     synchronous messaging permits a message sender to operate without
-     the cooperation of the message recipient.  Thus, implementing
-     message sending is simpler and processing is more efficiently
-     overlapped with it.  Remote method invocation is also supported.
-     The library was developed at the Los Alamos National Laboratory's
-     Advanced Computing Laboratory.</para>
- 
-     <para>&cheetah;'s messaging is implemented using an underlying
-     messaging library such as the Message Passing Interface (&mpi;)
-     Communications Library (FIXME: xref linkend="mpi99", <ulink
-     url="http://www-unix.mcs.anl.gov/mpi/"></ulink>) or the &mm;
-     Shared Memory Library.  &mpi; works on a wide variety of platforms
-     and has achieved widespread usage.  &mm; works under Unix on any
-     computer with shared memory.  Both libraries are available for
-     free.  The instructions below work for whichever library you
-     choose.</para>
- 
-     <para>We describe how to download and install &cheetah;.
-      <orderedlist spacing="compact">
-       <listitem>
-        <para>Download the library from the &pooma; Download page
-        available off the &pooma; home page (&poomaHomePage;).</para>
-       </listitem>
-       <listitem>
-        <para>Extract the source code using <command>tar xzvf
-        cheetah-1.0.tgz</command>.  Move into the resulting source code
-        directory <filename
-        class="directory">cheetah-1.0</filename>.</para>
-       </listitem>
-       <listitem>
-        <para>Edit a configuration file corresponding to your operating
-        system and compiler.  These <filename
-        class="libraryfile">.conf</filename> files are located in the
-        <filename class="directory">config</filename> directory.  For
-        example, to use &gcc; with the &linux; operating system, use
-        <filename
-        class="libraryfile">config/LINUXGCC.conf</filename>.</para>
- 
-        <para>The configuration file usually does not need
-        modification.  However, if you are using &mm;, ensure
-        <varname>shmem_default_dir</varname> specifies its location.
-        For example, the &author; modified the value to
-        <statement>"/home/oldham/pooma/mm-1.1.3"</statement>.</para>
-       </listitem>
-       <listitem>
-        <para>Prepare to compile the source code by configuring it
-        using the <command>configure</command> command.  Specify the
-        configuration file using the <command>--arch</command> option.
-        Its argument should be the configuration file's name, omitting
-        its <filename class="libraryfile">.conf</filename> suffix.  For
-        example, <command>--arch LINUXGCC</command>.  Some other
-        options include
-        <variablelist>
-        <varlistentry>
- 	<term>--help</term>
- 	<listitem>
- 	 <para>lists all the available options</para>
- 	</listitem>
-        </varlistentry>
-        <varlistentry>
- 	<term>--shmem --nompi</term>
- 	<listitem>
- 	 <para>indicates use of &mm;, not &mpi;</para>
- 	</listitem>
-        </varlistentry>
-        <varlistentry>
- 	<term>--mpi --noshmem</term>
- 	<listitem>
- 	 <para>indicates use of &mpi;, not &mm;</para>
- 	</listitem>
-        </varlistentry>
-        <varlistentry>
- 	<term>--opt</term>
- 	<listitem>
- 	 <para>causes the compiler to produce optimized source code</para>
- 	</listitem>
-        </varlistentry>
-        <varlistentry>
- 	<term>--noex</term>
- 	<listitem>
- 	 <para>prevents use of &cc; exceptions</para>
- 	</listitem>
-        </varlistentry>
-        <varlistentry>
- 	<term>--prefix <replaceable>directory</replaceable></term>
- 	<listitem>
- 	 <para>specifies the installation directory where the
- 	   library will be copied rather than the default.</para>
- 	</listitem>
-        </varlistentry>
-       </variablelist>
-         For example, the &author; uses <command>./configure --arch
-         LINUXGCC --shmem --nompi --noex --prefix
-         ${HOME}/pooma/cheetah-1.0 --opt</command>.  The
-         <command>--arch LINUXGCC</command> indicates use of &gcc;
-         under a &linux; operating system.  The &mm; library is used,
-         but &cc; exceptions are not.  The latter choice matches
-         &pooma;'s default choice.  The library will be installed in
-         the <filename
- 		      class="directory">${HOME}/pooma/cheetah-1.0</filename>.
-         Finally, the library code will be optimized, hopefully running
-         faster than unoptimized code.</para>
-       </listitem>
-       <listitem>
-        <para>Follow the directions printed by
-        <command>configure</command>: Change directories to the
-        <filename class="directory">lib</filename> subdirectory named
-        by the <command>--arch</command> argument and then type
-        <command>make</command> to compile the source code and create
-        the library.</para>
-       </listitem>
-       <listitem>
-        <para>Optionally ensure the library works correctly by issuing
-        the <command>make tests</command> command.</para>
-       </listitem>
-       <listitem>
-        <para>Install the library by issuing the <command>make
-        install</command> command.  This copies the library files to
-        the installation directory.  The <filename
-        class="directory">cheetah-1.0</filename> directory containing
-        the source code may now be removed.</para>
-       </listitem>
-      </orderedlist>
-    </para>
-    </section>
- 
-    <section id="installation-distributed_computing-pooma">
-     <title>Configuring &pooma; When Using &cheetah;</title>
- 
-     <para>To use &pooma; with &cheetah;, one must tell &pooma; the
-     location of the &cheetah; library using the
-     <command>--messaging</command> configuration option.  To do this,
-      <orderedlist spacing="compact">
-       <listitem>
-        <para>Set the &cheetah; directory environment variable
-         <envar>CHEETAHDIR</envar> to the directory containing the
-         installed &cheetah; library.  For
-         example, <command>declare -x
-         CHEETAHDIR=${HOME}/pooma/cheetah-1.0</command> specifies the
-         installation directory used in the previous section.</para>
-       </listitem>
-       <listitem>
-        <para>When configuring &pooma;, specify the
-        <command>--messaging</command> option.  For example,
-        <command>./configure --arch LINUXgcc --opt
-        --messaging</command> configures for &linux;, &gcc;, and an
-        optimized library using &cheetah;.</para>
-       </listitem>
-      </orderedlist>
-     </para>
-    </section>
- 
- <!-- HERE -->
-   </section>
-  </appendix>
- 
- 
-  <appendix id="compilation_errors">
-   <title>Dealing with Compilation Errors</title>
- 
-   <para>Base this low-priority section on <filename
-   class="libraryfile">errors.html</filename>.  QUESTION: Where is
-   <filename class="libraryfile">errors.html</filename>?</para>
- 
-  </appendix>
- 
- 
-  <appendix id="tmp_appendix">
-   <title>TMP: Notes to Myself</title>
- 
-   <section id="tmp-miscellaneous">
-    <title>Miscellaneous</title>
- 
-    <orderedlist>
-     <listitem>
-      <para>QUESTION: How do I know when to use a type name versus just
-      the concept?  For example, when do I use <quote>array</quote>
-      versus &array;?</para>
-     </listitem>
- 
-     <listitem>
-      <para>Krylov solvers are described in Section&nbsp;3.5.2 of
-      <filename>papers/pooma.ps</filename>.</para>
-     </listitem>
- 
-     <listitem>
-      <para>Section&nbsp;5, "The Polygon Overlay Problem," describes
-      porting an <acronym>ANSI</acronym> &c; program to &pooma;.</para>
-     </listitem>
- 
-     <listitem>
-      <para>A good example book: <emphasis>STL Tutorial and Reference
-      Guide: &cc; Programming with the Standard Template
-      Library</emphasis>, second edition, by David&nbsp;R. Musser,
-      Gillmer&nbsp;J. Derge, and Atul Sanai, ISBN 0-201-37923-6,
-      QA76.73.C153.M87 2001.</para>
-     </listitem>
- 
-     <listitem>
-      <para>One STL reference book listed functions in margin notes,
-      easing finding material.  Do this.</para>
-     </listitem>
- 
-     <listitem>
-      <para>QUESTION: Does Berna Massingill at Trinity University have
-      any interest ior access to any parallel computers?</para>
-     </listitem>
-    </orderedlist>
- 
-   </section>
- 
- 
-   <section id="tmp-html_tutorials">
-    <title>Existing HTML Tutorials</title>
- 
-    <para>All these tutorials are out-of-date, but the ideas and text
-    may still be relevant.</para>
- 
-    <variablelist>
-     <varlistentry><term><filename
- 				  class="libraryfile">index.html</filename></term>
-      <listitem><para>list of all tutorials.  No useful
-     material.</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
- 				  class="libraryfile">introduction.html</filename></term>
-      <listitem><para>data-parallel Laplace solver using Jacobi
-     iteration ala <command>Doof2d</command></para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
- 				  class="libraryfile">background.html</filename></term>
-      <listitem><para>short, indirect introduction to &pete;; parallel
-     execution model; &cc;; templates; &stl;; expression
-     templates</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
- 				  class="libraryfile">tut-01.html</filename></term>
-      <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
- 				  class="libraryfile">Layout.html</filename></term>
-      <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
- 				  class="libraryfile">parallelism.html</filename></term>
-      <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
- 				  class="libraryfile">self-test.html</filename></term>
-      <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">threading.html</filename></term>
-     <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">tut-03.html</filename></term>
-     <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">tut-04.html</filename></term>
-     <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">tut-05.html</filename></term>
-     <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">tut-06.html</filename></term>
-     <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">tut-07.html</filename></term>
-     <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">tut-08.html</filename></term>
-     <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">tut-09.html</filename></term>
-     <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">tut-10.html</filename></term>
-     <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">tut-11.html</filename></term>
-     <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">tut-12.html</filename></term>
-     <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-     <varlistentry><term><filename
-     class="libraryfile">tut-13.html</filename></term>
-     <listitem><para>UNFINISHED</para></listitem>
-     </varlistentry>
-    </variablelist>
- 
-   </section>
- 
-  </appendix>
- 
- 
-  <!-- Bibliography -->
- 
-  <bibliography id="bibliography">
-   <title>Bibliography</title>
- 
-   <para>FIXME: How do I process these entries?</para>
- 
-   <biblioentry>
-    <abbrev>mpi99</abbrev>
-    <authorgroup>
-     <author>
-      <firstname>William</firstname><surname>Gropp</surname>
-     </author>
-     <author>
-      <firstname>Ewing</firstname><surname>Lusk</surname>
-     </author>
-     <author>
-      <firstname>Anthony</firstname><surname>Skjellum</surname>
-     </author>
-    </authorgroup>
-    <copyright>
-     <year>1999</year>
-     <holder>Massachusetts Institute of Technology</holder>
-    </copyright>
-    <isbn>0-262-57132-3</isbn>
-    <publisher>
-     <publishername>The MIT Press</publishername>
-     <address>Cambridge, MA</address>
-    </publisher>
-    <title>Using MPI</title>
-    <subtitle>Portable Parallel Programming with the Message-Passing Interface</subtitle>
-    <edition>second edition</edition>
-   </biblioentry>
-  </bibliography>
- 
- 
-  <!-- Glossary -->
- 
-  <glossary id="glossary">
-   <title>Glossary</title>
- 
-   <para>ADD: Make sure all entries are indexed and perhaps point back
-   to their first use.  WARNING: This is constructed by hand so it is
-   likely to be full of inconsistencies and errors.</para>
- 
-   <glossdiv>
-    <title>S</title>
- 
-    <glossentry id="glossary-suite_name">
-     <glossterm>Suite Name</glossterm>
-     <glossdef>
-      <para>An arbitrary string denoting a particular toolkit
-      configuration.  For example, the string
-      <quote>SUNKCC-debug</quote> might indicate a configuration for
-      the <trademark>Sun</trademark> <productname>Solaris</productname>
-      operating system and the &kcc; &cc; compiler with debugging
-      support.  By default, the suite name it is equal to the
-      configuration's architecture name.</para>
-     </glossdef>
-    </glossentry>
-   </glossdiv>
-  </glossary>
-  
- 
-  <!-- Index -->
- 
-   &genindex.sgm;
- 
- </book>
--- 0 ----
Index: tutorial.xml
===================================================================
RCS file: tutorial.xml
diff -N tutorial.xml
*** /dev/null	Fri Mar 23 21:37:44 2001
--- tutorial.xml	Tue Dec 11 13:31:10 2001
***************
*** 0 ****
--- 1,1051 ----
+ <chapter id="tutorial">
+  <title>A Tutorial Introduction</title>
+ 
+  <para>UPDATE: In the following paragraph, fix the cross-reference
+  to the actual section.</para>
+ 
+  <para>&pooma; provides different containers and processor
+  configurations and supports different implementation styles, as
+  described in <xref linkend="introduction"></xref>.  In this
+  chapter, we present several different implementations of the
+  &doof2d; two-dimensional diffusion simulation program:
+   <itemizedlist spacing="compact">
+    <listitem>
+     <para>a C-style implementation omitting any use of &pooma;
+     computing each array element individually,</para>
+    </listitem>
+    <listitem>
+     <para>a &pooma; &array; implementation computing each array
+     element individually,</para>
+    </listitem>
+    <listitem>
+     <para>a &pooma; &array; implementation using data-parallel
+     statements,</para>
+    </listitem>
+    <listitem>
+     <para>a &pooma; &array; implementation using stencils, which
+     support local computations,</para>
+    </listitem>
+    <listitem>
+     <para>a stencil-based &pooma; &array; implementation supporting
+     computation on multiple processors</para>
+    </listitem>
+    <listitem>
+     <para>a &pooma; &field; implementation using data-parallel
+     statements, and</para>
+    </listitem>
+    <listitem>
+     <para>a data-parallel &pooma; &field; implementation for
+     multi-processor execution.</para>
+    </listitem>
+   </itemizedlist>
+  </para>
+  <para>These illustrate the &array;, &field;, &engine;, layout,
+  mesh, and domain data types.  They also illustrate various
+  immediate computation styles (element-wise accesses, data-parallel
+  expressions, and stencil computation) and various processor
+  configurations (one sequential processor and multiple
+  processors).</para>
+ 
+  <figure float="1" id="tutorial-doof2d_averagings">
+   <title>&doof2d; Averagings</title>
+   <mediaobject>
+    <imageobject>
+     <imagedata fileref="figures/doof2d.201" format="EPS" align="center"></imagedata>
+    </imageobject>
+    <textobject>
+     <phrase>The Initial Configuration</phrase>
+    </textobject>
+   </mediaobject>
+   <mediaobject>
+    <imageobject>
+     <imagedata fileref="figures/doof2d.202" format="EPS"></imagedata>
+    </imageobject>
+    <textobject>
+     <phrase>After the First Averaging</phrase>
+    </textobject>
+   </mediaobject>
+   <mediaobject>
+    <imageobject>
+     <imagedata fileref="figures/doof2d.203" format="EPS"></imagedata>
+    </imageobject>
+    <textobject>
+     <phrase>After the Second Averaging</phrase>
+    </textobject>
+   </mediaobject>
+  </figure>
+ 
+  <para>The &doof2d; diffusion program starts with a two-dimensional
+  grid of values.  To model an initial density, all grid values are
+  zero except for one nonzero value in the center.  Each averaging,
+  each grid element, except the outermost ones, updates its value by
+  averaging its value and its eight neighbors.  To avoid overwriting
+  grid values before all their uses occur, we use two arrays, reading
+  the first and writing the second and then reversing their roles
+  within each iteration.</para>
+ 
+  <para>Figure <xref linkend="tutorial-doof2d_averagings"></xref>
+  illustrates the averagings.  Initially, only the center element has
+  nonzero value.  To form the first averaging, each element's new
+  value equals the average of its and its neighbors' previous values.
+  Thus, the initial nonzero value spreads to a three-by-three grid.
+  The averaging continues, spreading to a five-by-five grid of
+  nonzero values.  Values in outermost grid cells are always
+  zero.</para>
+ 
+  <para>Before presenting various implementations of %doof2d;, we
+  explain how to install the &poomaToolkit;.</para>
+ 
+  <para>REMOVE: &doof2d; algorithm and code is illustrated in
+  Section&nbsp;4.1 of
+  <filename>pooma-publications/pooma.ps</filename>.  It includes a
+  figure illustrating parallel communication of data.</para>
+ 
+  <section id="tutorial-installation">
+   <title>Installing &pooma;</title>
+ 
+   <para>ADD: How does one install &pooma; using Windows or Mac?</para>
+ 
+   <para>UPDATE: Make a more recent &pooma; source code file
+   available on &poomaDownloadPage;.  For example,
+   <quote>LINUXgcc.conf</quote> is not available.</para>
+ 
+   <para>In this section, we describe how to obtain, build, and
+   install the &poomaToolkit;.  We focus on installing under the
+   Unix operating system.  Instructions for installing on computers
+   running Microsoft Windows or MacOS, as well as more extensive
+   instructions for Unix, appear in <xref
+   linkend="installation"></xref>.</para>
+ 
+   <para>Obtain the &pooma; source code <filename
+   path="http://www.codesourcery.com/pooma/downloads_folder/">&poomaSourceFile;</filename>
+   from the &pooma; download page (&poomaDownloadPage;) available off
+   the &pooma; home page (&poomaHomePage;).  The <quote>tgz</quote>
+   indicates this is a compressed tar archive file.  To extract the
+   source files, use <command>tar xzvf &poomaSourceFile;</command>.
+   Move into the source code directory <filename
+   class="directory">&poomaSource;</filename> directory; e.g.,
+   <command>cd &poomaSource;</command>.</para>
+ 
+   <para>Configuring the source code prepares the necessary paths for
+   compilation.  First, determine a configuration file in
+   corresponding to your operating system and compiler in the
+   <filename class="directory">config/arch/</filename> directory.
+   For example, <filename
+   class="libraryfile">LINUXgcc.conf</filename> supports compiling
+   under a &linux; operating system with &gcc; and <filename
+   class="libraryfile">SGI64KCC.conf</filename> supports compiling
+   under a 64-bit <application>SGI</application> Unix operating
+   system with &kcc;.  Then, configure the source code:
+   <command>./configure &dashdash;arch LINUXgcc &dashdash;opt &dashdash;suite
+   LINUXgcc-opt</command>.  The architecture argument to the
+   <command>&dashdash;arch</command> option is the name of the corresponding
+   configuration file, omitting its <filename
+   class="libraryfile">.conf</filename> suffix.  The
+   <command>&dashdash;opt</command> indicates the &poomaToolkit; will
+   contain optimized source code, which makes the code run more
+   quickly but may impede debugging.  Alternatively, the
+   <command>&dashdash;debug</command> option supports debugging.  The
+   <glossterm linkend="glossary-suite_name">suite name</glossterm>
+   can be any arbitrary string.  We chose
+   <command>LINUXgcc-opt</command> to remind us of the architecture
+   and optimization choice.  <filename
+   class="libraryfile">configure</filename> creates subdirectories
+   named by the suite name <quote>LINUXgcc-opt</quote> for use when
+   compiling the source files.  Comments at the beginning of
+   <filename
+   class="libraryfile">lib/<replaceable>suiteName</replaceable>/PoomaConfiguration.h</filename>
+   record the configuration arguments.</para>
+ 
+   <para>To compile the source code, set the
+   <envar>POOMASUITE</envar> environment variable to the suite name
+   and then type <command>make</command>.  To set the environment
+   variable for the <application>bash</application> shell use
+   <command>export
+   POOMASUITE=<replaceable>suiteName</replaceable></command>,
+   substituting the suite name's
+   <replaceable>suiteName</replaceable>.  For the
+   <application>csh</application> shell, use <command>setenv
+   POOMASUITE LINUXgcc-opt</command>.  Issuing the
+   <command>make</command> command compiles the &pooma; source code
+   files to create the &pooma; library.  The &pooma; makefiles assume
+   the <trademark>GNU</trademark> &make; so substitute the proper
+   command if necessary.  The &pooma; library can be found in, e.g.,
+   <filename
+   class="libraryfile">lib/LINUXgcc-opt/libpooma-gcc.a</filename>.</para>
+  </section>
+ 
+  <section id="tutorial-hand_coded">
+   <title>Hand-Coded Implementation</title>
+ 
+   <para>Before implementing &doof2d; using the &poomaToolkit;, we
+   present a hand-coded implementation of &doof2d;.  See <xref
+   linkend="tutorial-hand_coded-doof2d"></xref>.  After querying the
+   user for the number of averagings, the arrays' memory is
+   allocated.  Since the arrays' size is not known at compile time,
+   the arrays are accesses via pointers to allocated dynamic memory.
+   This memory is deallocated at the program's end to avoid memory
+   leaks.  The arrays are initialized with initial conditions.  For
+   the <varname>b</varname> array, all values except the central ones
+   have nonzero values.  Only the outermost values of the
+   <varname>a</varname> array need be initialized to zero, but we
+   instead initialize them all using the loop used by
+   <varname>b</varname>.</para>
+ 
+   <para>The simulation's kernel consists of triply nested loops.
+   The outermost loop controls the number of iterations.  The inner
+   nested loops iterate through the arrays' elements, excepting the
+   outermost elements; note the loop indices range from 1 to n-2
+   while the array indices range from 0 to n-1.  Each
+   <varname>a</varname> value is assigned the average of its
+   corresponding value in <varname>b</varname> and the latter's
+   neighbors.  Values in the two-dimensional grids are accessed using
+   two sets of brackets, e.g., <statement>a[i][j]</statement>.  After
+   assigning values to <varname>a</varname>, a second averaging reads
+   values in <varname>a</varname>, writing values in
+   <varname>b</varname>.</para>
+ 
+   <para>After the kernel finishes, the final central value is
+   printed.  If the desired number of averagings is even, the value
+   in <varname>b</varname> is printed; otherwise, the value in
+   <varname>a</varname> is used.  Finally, the dynamically-allocated
+   memory must be freed to avoid memory leaks.</para>
+ 
+   <example id="tutorial-hand_coded-doof2d">
+    <title>Hand-Coded Implementation of &doof2d;</title>
+    &doof2d-c-element;
+    <calloutlist>
+     <callout arearefs="tutorial-hand_coded-doof2d-nuaveragings">
+      <para>The user specifies the desired number of averagings.</para>
+     </callout>
+     <callout arearefs="tutorial-hand_coded-doof2d-array_storage">
+      <para>These variables point to the two-dimensional,
+      dynamically-allocated grids so we use a pointer to a pointer to
+      a &double;.</para>
+     </callout>
+     <callout arearefs="tutorial-hand_coded-doof2d-grid_size">
+      <para>The user enters the desired grid size.  The grid will be
+      a square with <varname>n</varname> by <varname
+      lang="c++">n</varname> grid cells.</para>
+     </callout>
+     <callout arearefs="tutorial-hand_coded-doof2d-allocation">
+      <para>Memory for the arrays is allocated.  By default, the
+      array indices are zero-based.</para>
+     </callout>
+     <callout arearefs="tutorial-hand_coded-doof2d-initialization">
+      <para>Initially, all grid values are zero except for the one
+      nonzero value at the center of the second array.  Array
+      positions are indicated using two brackets, e.g.,
+      <statement>a[i][j]</statement>.  A better implementation might
+      initialize only the outermost values of the
+      <varname>a</varname> array.</para>
+     </callout>
+     <callout arearefs="tutorial-hand_coded-doof2d-constants">
+      <para>These constants indicate the number of iterations, and
+      the average weighting.</para>
+     </callout>
+     <callout arearefs="tutorial-hand_coded-doof2d-first_write">
+      <para>Each <varname>a</varname> value, except an outermost one,
+      is assigned the average of its analogous <varname>b</varname>
+      value and that value's neighbors.  Note the loop indices ensure
+      the outermost values are not changed.  The
+      <varname>weight</varname>'s value ensures the computation is an
+      average.</para>
+     </callout>
+     <callout arearefs="tutorial-hand_coded-doof2d-second_write">
+      <para>The second averaging computes <varname>b</varname>'s
+      values using values stored in <varname>a</varname>.</para>
+     </callout>
+     <callout arearefs="tutorial-hand_coded-doof2d-answer">
+      <para>After the averagings finish, the central value is printed.</para>
+     </callout>
+     <callout arearefs="tutorial-hand_coded-doof2d-deallocation">
+      <para>The dynamically-allocated memory must be deallocated to
+      avoid memory leaks.</para>
+     </callout>
+    </calloutlist>
+   </example>
+ 
+   <para>To compile the executable, change directories to the &pooma;
+   <filename
+   class="directory">&poomaExampleDirectory;/Doof2d</filename>
+   directory.  Ensure the <envar>POOMASUITE</envar> environment
+   variable specifies the desired suite name
+   <replaceable>suiteName</replaceable>, as we did when compiling
+   &pooma; in the previous section <xref
+   linkend="tutorial-installation"></xref>.  Issuing the
+   <command>make Doof2d-C-element</command> command creates the
+   executable
+   <command><replaceable>suiteName</replaceable>/Doof2d-C-element</command>.</para>
+ 
+   <para>When running the executable, specify the desired a
+   nonnegative number of averagings and the nonnegative number of
+   grid cells along any dimension.  The resulting grid has the same
+   number of cells along each dimension.  After the executable
+   finishes, the resulting value of the central element is
+   printed.</para>
+  </section>
+ 
+ 
+  <section id="tutorial-array_elementwise">
+   <title>Element-wise &array; Implementation</title>
+ 
+   <para>The simplest way to use the &poomaToolkit; is to
+   use the &pooma; &array; class instead of &c; arrays.  &array;s
+   automatically handle memory allocation and deallocation, support a
+   wider variety of assignments, and can be used in expressions.
+   <xref linkend="tutorial-array_elementwise-doof2d"></xref>
+   implements &doof2d; using &array;s and element-wise accesses.
+   Since the same algorithm is used as <xref
+   linkend="tutorial-hand_coded-doof2d"></xref>, we will concentrate
+   on the differences.</para>
+ 
+   <example id="tutorial-array_elementwise-doof2d">
+    <title>Element-wise &array; Implementation of &doof2d;</title>
+    &doof2d-array-element;
+    <calloutlist>
+     <callout arearefs="tutorial-array_elementwise-doof2d-header">
+      <para>To use &pooma; &array;s, the <filename
+      class="headerfile">Pooma/Arrays.h</filename> must be included.</para>
+     </callout>
+     <callout arearefs="tutorial-array_elementwise-doof2d-pooma_initialize">
+      <para>The &poomaToolkit; structures must be constructed before
+      their use.</para>
+     </callout>
+     <callout arearefs="tutorial-array_elementwise-doof2d-domain">
+      <para>Before creating an &array;, its domain must be specified.
+      The <varname>N</varname> interval represents the
+      one-dimensional integral set {0, 1, 2, &hellip;, n-1}.  An
+      <type>Interval&lt;2&gt;</type> object represents the entire
+      two-dimensional index domain.</para>
+     </callout>
+     <callout arearefs="tutorial-array_elementwise-doof2d-array_creation">
+      <para>An &array;'s template parameters indicate its dimension,
+      its value type, and how the values will be stored or computed.
+      The &brick; &engine; type indicates values will be directly
+      stored.  It is responsible for allocating and deallocating
+      storage so <statement>new</statement> and
+      <statement>delete</statement> statements are not necessary.
+      The <varname>vertDomain</varname> specifies the array index
+      domain.</para>
+     </callout>
+     <callout arearefs="tutorial-array_elementwise-doof2d-initialization">
+      <para>The first statement initializes all &array; values to the
+      same scalar value.  This is possible because each &array;
+      <quote>knows</quote> its domain.  The second statement
+      illustrates &array; element access.  Indices, separated by
+      commas, are surrounded by parentheses rather than surrounded by
+      square brackets (<statement>[]</statement>).</para>
+     </callout>
+     <callout arearefs="tutorial-array_elementwise-doof2d-first_write">
+      <para>&array; element access uses parentheses, rather than
+      square brackets</para>
+     </callout>
+     <callout arearefs="tutorial-array_elementwise-doof2d-deallocation">
+      <para>Since &array;s are first-class objects, they
+      automatically deallocate any memory they require, eliminating
+      memory leaks.</para>
+     </callout>
+     <callout arearefs="tutorial-array_elementwise-doof2d-pooma_finish">
+      <para>The &poomaToolkit; structures must be destructed after
+      their use.</para>
+     </callout>
+    </calloutlist>
+   </example>
+ 
+   <para>We describe the use of &array; and the &poomaToolkit; in
+   <xref linkend="tutorial-array_elementwise-doof2d"></xref>.
+   &array;s, declared in the <filename
+   class="headerfile">Pooma/Arrays.h</filename>, are first-class
+   objects.  They <quote>know</quote> their index domain, can be used
+   in expressions, can be assigned scalar and array values, and
+   handle their own memory allocation and deallocation.</para>
+ 
+   <para>The creation of the <varname>a</varname> and
+   <varname>b</varname> &array;s requires an object specifying their
+   index domains.  Since these are two-dimensional arrays, their
+   index domains are also two dimensional.  The two-dimensional
+   <type>Interval&lt;2&gt;</type> object is the Cartesian product of
+   two one-dimensional <type>Interval&lt;1&gt;</type> objects, each
+   specifying the integral set {0, 1, 2, &hellip;, n-1}.</para>
+ 
+   <para>An &array;'s template parameters indicate its dimension, the
+   type of its values, and how the values are stored.  Both
+   <varname>a</varname> and <varname>b</varname> are two-dimension
+   arrays storing &double;s so their <varname>dimension</varname>
+   is&nbsp;2 and its element type is &double;.  An &engine; stores an
+   &array;'s values.  For example, a &brick; &engine; explicitly
+   stores all values.  A &compressiblebrick; &engine; also explicitly
+   stores values if more than value is present, but, if all values
+   are the same, storage for just that value is required.  Since an
+   engine can store its values any way it desires, it might instead
+   compute its values using a function or compute the values stored
+   in separate engines.  In practice, most explicitly specified
+   &engine;s are either &brick; or &compressiblebrick;.</para>
+ 
+   <para>&array;s support both element-wise access and scalar
+   assignment.  Element-wise access uses parentheses, not square
+   brackets.  For example, <statement>b(n/2,n/2)</statement>
+   specifies the central element.  The scalar assignment <statement>b
+   = 0.0</statement> assigns the same 0.0 value to all array
+   elements.  This is possible because the array knows the extent of
+   its domain.</para>
+ 
+   <para>Any program using the &poomaToolkit; must initialize the
+   toolkit's data structures using
+   <statement>Pooma::initialize(argc,argv)</statement>.  This
+   extracts &pooma;-specific command-line options from the
+   command-line arguments in <varname>argv</varname> and initializes
+   the inter-processor communication and other data structures.  When
+   finished, <statement>Pooma::finalize()</statement> ensures all
+   computation has finished and the communication and other data
+   structures are destructed.</para>
+  </section>
+ 
+ 
+  <section id="tutorial-array_data_parallel">
+   <title>Data-Parallel &array; Implementation</title>
+ 
+   <para>&pooma; supports data-parallel &array; accesses.  Many
+   algorithms are more easily expressed using data-parallel
+   expressions.  Also, the &poomaToolkit; might be able to reorder
+   the data-parallel computations to be more efficient or distribute
+   them among various processors.  In this section, we concentrate
+   the differences between the data-parallel implementation of
+   &doof2d; listed in <xref
+   linkend="tutorial-array_parallel-doof2d"></xref> and the
+   element-wise implementation listed in the previous section <xref
+   linkend="tutorial-array_elementwise"></xref>.</para>
+ 
+   <example id="tutorial-array_parallel-doof2d">
+    <title>Data-Parallel &array; Implementation of &doof2d;</title>
+    &doof2d-array-parallel;
+    <calloutlist>
+     <callout arearefs="tutorial-array_parallel-doof2d-blockAndEvaluate">
+      <para>&pooma; may reorder computation of statements.  Calling
+      <function>Pooma::blockAndEvaluate</function> ensures all
+      computation finishes before accessing a particular array
+      element.</para>
+     </callout>
+     <callout arearefs="tutorial-array_parallel-doof2d-innerdomain">
+      <para>These variables specify one-dimensional domains {1, 2,
+      &hellip;, n-2}.  Their Cartesian product specifies the domain
+      of the array values that are modified.</para>
+     </callout>
+     <callout arearefs="tutorial-array_parallel-doof2d-first_write">
+      <para>Data-parallel expressions replace nested loops and array
+      element accesses.  For example, <statement>a(I,J)</statement>
+      represents the subset of the <varname>a</varname> array having
+      a domain equal to the Cartesian product of <varname>I</varname>
+      and <varname>J</varname>.  Intervals can shifted by an additive
+      or multiplicative constant.</para>
+     </callout>
+    </calloutlist>
+   </example>
+ 
+   <para>Data-parallel expressions apply domain objects to containers
+   to indicate a set of parallel expressions.  For example, in the
+   program listed above, <statement>a(I,J)</statement> specifies all
+   of <varname>a</varname> array excepting the outermost elements.
+   The array's <varname>vertDomain</varname> domain consists of the
+   Cartesian product of {0, 1, 2, &hellip;, n-1} and itself, while
+   <varname>I</varname> and <varname>J</varname> each specify {1, 2,
+   &hellip;, n-2}.  Thus, <statement>a(I,J)</statement> is the subset
+   with a domain of the Cartesian product of {1, 2, &hellip;, n-2}
+   and itself.  It is called a <firstterm>view</firstterm> of an
+   array.  It is itself an array, with a domain and supporting
+   element access, but its storage is the same as
+   <varname>a</varname>'s.  Changing a value in
+   <statement>a(I,J)</statement> also changes the same value in
+   <varname>a</varname>.  Changing a value in the latter also changes
+   the former if the value is not one of <varname>a</varname>'s
+   outermost elements.  The expression
+   <statement>b(I+1,J+1)</statement> indicates the subset of
+   <varname>b</varname> with a domain consisting of the Cartesian
+   product of {2, 3, &hellip;, n-1}, i.e., the same domain as
+   <statement>a(I,J)</statement> but shifted up one unit and to the
+   right one unit.  Only an &interval;'s value, not its name, is
+   important.  Thus, all uses of <varname>J</varname> in this program
+   could be replaced by <varname>I</varname> without changing the
+   semantics.</para>
+ 
+   <figure float="1" id="tutorial-array_parallel-doof2d-adding_arrays">
+    <title>Adding &array;s</title>
+    <mediaobject>
+     <imageobject>
+      <imagedata fileref="figures/doof2d.210" format="EPS" align="center"></imagedata>
+     </imageobject>
+     <textobject>
+      <phrase>Adding two arrays with different domains.</phrase>
+     </textobject>
+     <caption>
+      <para>When adding arrays, values in corresponding positions are
+      added even if they have different indices, indicated by the
+      small numbers adjacent to the arrays.</para>
+     </caption>
+    </mediaobject>
+   </figure>
+ 
+   <para>The statement assigning to <statement>a(I,J)</statement>
+   illustrates that &array;s may participate in expressions.  Each
+   addend is a view of an array, which is itself an array.  Each view
+   has the same domain size so their sum can be formed by
+   corresponding elements of each array.  For example, the lower,
+   left element of the result equals the sum of the lower, left
+   elements of the addend arrays.  For the computation, indices are
+   ignored; only the relative positions within each domain are used.
+   <xref
+   linkend="tutorial-array_parallel-doof2d-adding_arrays"></xref>
+   illustrates adding two arrays with different domain indices.  The
+   indices are indicated by the small numbers to the left and the
+   bottom of the arrays.  Even though 9 and 3 have different indices
+   (1,1) and (2,0), they are added to each other because they have
+   the same relative positions within the addends.</para>
+ 
+   <para>Just before accessing individual &array; values, the code
+   contains calls to <function>Pooma::blockAndEvaluate</function>.
+   &pooma; may reorder computation or distribute them among various
+   processors.  Before reading an individual &array; value, calling
+   the function ensures all computations affecting its value have
+   finished, i.e., it has the correct value.  Calling this function
+   is necessary only when accessing individual array elements because
+   &pooma; cannot determine when to call the function itself. For
+   example, before printing an array, &pooma; will call
+   <function>blockAndEvaluate</function> itself.</para>
+  </section>
+ 
+ 
+  <section id="tutorial-array_stencil">
+   <title>Stencil &array; Implementation</title>
+ 
+   <para>Many computations are local, computing a &array;'s value by
+   using close-by &array; values.  Encapsulating this computation in
+   a stencil can yield faster code because the compiler can determine
+   all accesses come from the same array.  Each stencil consists of a
+   function object and an indication of the stencil's extent.</para>
+ 
+   <example id="tutorial-array_stencil-doof2d">
+    <title>Stencil &array; Implementation of &doof2d;</title>
+    &doof2d-array-stencil;
+    <calloutlist>
+     <callout arearefs="tutorial-array_stencil-doof2d-stencil">
+      <para>A stencil is a function object implementing a local
+      operation on an &array;.</para>
+     </callout>
+     <callout
+      arearefs="tutorial-array_stencil-doof2d-stencil_operator">
+      <para>&pooma; applies this function call
+      <function>operator()</function> to the interior domain of an
+      &array;.  Although not strictly necessary, the function's
+      template parameter <varname>C</varname> permits using this
+      stencil with &array;s and other containers.  The
+      <function>read</function> &array; member function supports only
+      reading values, not writing values, thus possibly permitting
+      faster access.</para>
+     </callout>
+     <callout arearefs="tutorial-array_stencil-doof2d-stencil_extent">
+      <para>These two functions indicate the stencil's size.  For
+      each dimension, the stencil extends one cell to the left of (or
+      below) its center and also one call to the right (or above) its
+      center.</para>
+     </callout>
+     <callout
+      arearefs="tutorial-array_stencil-doof2d-stencil_creation">
+      <para>Create the stencil.</para>
+     </callout>
+     <callout arearefs="tutorial-array_stencil-doof2d-first_write">
+      <para>Applying <varname>stencil</varname> to the
+      <varname>b</varname> array and a subset
+      <varname>interiorDomain</varname> of its domain yields an
+      array, which is assigned to a subset of <varname>a</varname>.
+      The stencil's function object is applied to each position in
+      the specified subset of <varname>b</varname>.</para>
+     </callout>
+    </calloutlist>
+   </example>
+ 
+   <para>Before we describe how to create a stencil, we describe how
+   to apply a stencil to an array, yielding values.  To compute the
+   value associated with index position (1,3), the stencil's center
+   is placed at (1,3).  The stencil's
+   <function>upperExtent</function> and
+   <function>lowerExtent</function> functions indicate which &array;
+   values the stencil's function will use.  See <xref
+   linkend="tutorial-array_stencil-doof2d-apply_stencil"></xref>.
+   Applying the stencil's function call
+   <function>operator()</function> yields the computed value.  To
+   compute multiple &array; values, apply a stencil to the array and
+   a domain object: <statement>stencil(b,
+   interiorDomain)</statement>.  This applies the stencil to each
+   position in the domain.  The user must ensure that applying the
+   stencil does not access nonexistent &array; values.</para>
+ 
+   <figure float="1" id="tutorial-array_stencil-doof2d-apply_stencil">
+    <title>Applying a Stencil to an &array;</title>
+    <mediaobject>
+     <imageobject>
+      <imagedata fileref="figures/doof2d.211" format="EPS" align="center"></imagedata>
+     </imageobject>
+     <textobject>
+      <phrase>Apply a stencil to position (1,3) of an array.</phrase>
+     </textobject>
+     <caption>
+      <para>To compute the value associated with index position (1,3)
+      of an array, place the stencil's center, indicated with dashed
+      lines, at the position.  The computation involves the array
+      values covered by the array and delineated by
+      <function>upperExtent</function> and
+      <function>lowerExtent</function>.</para>
+     </caption>
+    </mediaobject>
+   </figure>
+ 
+   <para>To create a stencil object, apply the &stencil; type to a
+   function object class.  For example,
+   <statement>Stencil&lt;DoofNinePt&gt; stencil</statement> declares
+   the <varname>stencil</varname> object.  The function object class
+   must define a function call <function>operator()</function> with a
+   container parameter and index parameters.  The number of index
+   parameters, indicating the stencil's center, must equal the
+   container's dimension.  For example, <type>DoofNinePt</type>
+   defines <methodname>operator()(const C& c, int i, int
+   j)</methodname>.  We templated the container type
+   <varname>C</varname> although this is not strictly necessary.  The
+   two index parameters <varname>i</varname> and <varname>j</varname>
+   ensure the stencil works with two-dimensional containers.  The
+   <methodname>lowerExtent</methodname> indicates how far to the left
+   (or below) the stencil extends beyond its center.  Its parameter
+   indicates a particular dimension.  Index parameters
+   <varname>i</varname> and <varname>j</varname> are in dimension 0
+   and&nbsp;1.  <methodname>upperExtent</methodname> serves an
+   analogous purpose.  The &poomaToolkit; uses these functions when
+   distribution computation among various processors, but it does not
+   use these functions to ensure nonexistent &array; values are not
+   accessed.  Caveat stencil user!</para>
+  </section>
+ 
+ 
+  <section id="tutorial-array_distributed">
+   <title>Distributed &array; Implementation</title>
+ 
+   <para>A &pooma; program can execute on one or multiple processors.
+   To convert a program designed for uniprocessor execution to a
+   program designed for multiprocessor execution, the programmer need
+   only specify how each container's domain should be split into
+   <quote>patches</quote>.  The &poomaToolkit; automatically
+   distributes the data among the available processors and handles
+   any required communication among processors.</para>
+ 
+   <example id="tutorial-array_distributed-doof2d">
+    <title>Distributed Stencil &array; Implementation of &doof2d;</title>
+    &doof2d-array-distributed;
+    <calloutlist>
+     <callout arearefs="tutorial-array_distributed-doof2d-io">
+      <para>Multiple copies of a distributed program may
+      simultaneously run, perhaps each having its own input and
+      output.  Thus, we use command-line arguments to pass input to
+      the program.  Using an &inform; object ensures only one program
+      produces output.</para>
+     </callout>
+     <callout arearefs="tutorial-array_distributed-doof2d-layout">
+      <para>The <type>UniformGridPartition</type> declaration
+      specifies how an array's domain will be partition, of split,
+      into patches.  Guard layers are an optimization that can reduce
+      data communication between patches.  The
+      <type>UniformGridLayout</type> declaration applies the
+      partition to the given domain, distributing the resulting
+      patches among various processors.</para>
+     </callout>
+     <callout arearefs="tutorial-array_distributed-doof2d-remote">
+      <para>The <type>MultiPatch</type> &engine; distributes requests
+      for &array; values to the associated patch.  Since a patch may
+      associated with a different processor, its
+      <quote>remote</quote> engine has type
+      <type>Remote&lt;Brick&gt;</type>.  &pooma; automatically
+      distributes the patches among available memories and
+      processors.</para>
+     </callout>
+     <callout
+      arearefs="tutorial-array_distributed-doof2d-first_write">
+      <para>The stencil computation, whether for one processor or
+      multiple processors, is the same.</para>
+     </callout>
+    </calloutlist>
+   </example>
+ 
+   <para>Supporting distributed computation requires only minor code
+   changes.  These changes specify how each container's domain is
+   distributed among the available processors and how input and
+   output occurs.  The rest of the program, including all the
+   computations, remains the same.  When running, the &pooma;
+   executable interacts with the run-time library to determine which
+   processors are available, distributes the containers' domains, and
+   automatically handles all necessary interprocessor communication.
+   The same executable runs on one or many processors.  Thus, the
+   programmer can write one program, debugging it on a uniprocessor
+   computer and running it on a supercomputer.</para>
+ 
+   <figure float="1" id="tutorial-array_distributed-doof2d-distributed_model">
+    <title>The &pooma; Distributed Computation Model</title>
+    <mediaobject>
+     <imageobject>
+      <imagedata fileref="figures/distributed.101" format="EPS" align="center"></imagedata>
+     </imageobject>
+     <textobject>
+      <phrase>the &pooma; distributed computation model.</phrase>
+     </textobject>
+     <caption>
+      <para>The &pooma; distributed computation model combines
+      partitioning containers' domains and the computer configuration
+      to create a layout.</para>
+     </caption>
+    </mediaobject>
+   </figure>
+ 
+   <para>&pooma;'s distributed computing model separates container
+   domain concepts from computer configuration concepts.  See <xref
+   linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>.
+   The program indicates how each container's domain will be
+   partitioned.  This process is represented in the upper left corner
+   of the figure.  A user-specified <firstterm>partition</firstterm>
+   specifies how to split the domain into pieces.  For example, the
+   illustrated partition splits the domain into three equal-sized
+   pieces along the x-dimension and two equal-sized pieces along the
+   y-dimension.  Thus, the domain is split into
+   <firstterm>patches</firstterm>.  The partition also specifies
+   external and internal guard layers.  A <firstterm>guard
+   layer</firstterm> is a domain surrounding a patch.  A patch's
+   computation only reads but does not write these guarded values.
+   An <firstterm>external guard layer</firstterm> conceptually
+   surrounds the entire container domain with boundary values whose
+   presence permits all domain computations to be performed the same
+   way even for values along the domain's edge.  An
+   <firstterm>internal guard layer</firstterm> duplicates values from
+   adjacent patches so communication need not occur during a patch's
+   computation.  The use of guard layers is an optimization; using
+   external guard layers eases programming and using internal guard
+   layers reduces communication among processors.  Their use is not
+   required.</para>
+ 
+   <para>The computer configuration of shared memory and processors
+   is determined by the run-time system.  See the upper right portion
+   of <xref
+   linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>.
+   A <firstterm>context</firstterm> is a collection of shared memory
+   and processors that can execute a program or a portion of a
+   program.  For example, a two-processor desktop computer might have
+   memory accessible to both processors so it is a context.  A
+   supercomputer consisting of desktop computers networked together
+   might have as many contexts as computers.  The run-time system,
+   e.g., the Message Passing Interface (&mpi;) Communications Library
+   (FIXME: xref linkend="mpi99", <ulink
+   url="http://www-unix.mcs.anl.gov/mpi/"></ulink>) or the &mm;
+   Shared Memory Library (<ulink
+   url="http://www.engelschall.com/sw/mm/"></ulink>), communicates
+   the available contexts to the executable.  &pooma; must be
+   configured for the particular run-time system.  See <xref
+   linkend="installation-distributed_computing"></xref>.</para>
+ 
+   <para>A <firstterm>layout</firstterm> combines patches with
+   contexts so the program can be executed.  If &distributedtag; is
+   specified, the patches are distributed among the available
+   contexts.  If &replicatedtag; is specified, each set of patches is
+   replicated among each context.  Regardless, the containers'
+   domains are now distributed among the contexts so the program can
+   run.  When a patch needs data from another patch, the &pooma;
+   toolkit sends messages to the desired patch uses a message-passing
+   library.  All such communication is automatically performed by the
+   toolkit with no need for programmer or user input.</para>
+ 
+   <para>FIXME: The two previous paragraphs demonstrate confusion
+   between <quote>run-time system</quote> and <quote>message-passing
+   library</quote>.</para>
+ 
+   <para>Incorporating &pooma;'s distributed computation model into a
+   program requires writing very few lines of code.  <xref
+   linkend="tutorial-array_distributed-doof2d"></xref> illustrates
+   this.  The <varname>partition</varname> declaration creates a
+   <type>UniformGridPartition</type> splitting each dimension of a
+   container's domain into equally-sized
+   <varname>nuProcessors</varname> pieces.  The first
+   <type>GuardLayers</type> argument specifies each patch will have
+   copy of adjacent patches' outermost values.  This may speed
+   computation because a patch need not synchronize its computation
+   with other patches' processors.  Since each value's computation
+   requires knowing its surrounding neighbors, the internal guard
+   layer is one layer deep.  The second <type>GuardLayers</type>
+   argument specifies no external guard layer.  External guard layers
+   simplify computing values along the edges of domains.  Since the
+   program already uses only the interior domain for computation, we
+   do not use this feature.</para>
+ 
+   <para>The <varname>layout</varname> declaration creates a
+   <type>UniformGridLayout</type> layout.  As <xref
+   linkend="tutorial-array_distributed-doof2d"></xref> illustrates,
+   it needs to know a container's domain, a partition, the computer's
+   contexts, and a &distributedtag; or &replicatedtag;.  These
+   comprise <varname>layout</varname>'s three parameters; the
+   contexts are implicitly supplied by the run-time system.</para>
+ 
+   <para>To create a distributed &array;, it should be created using
+   a &layout; object and have a &multipatch; engine.  Prior
+   implementations designed for uniprocessors constructed the
+   container using a &domain; object.  A distributed implementation
+   uses a &layout; object, which conceptually specifies a &domain;
+   object and its distribution throughout the computer.  A
+   &multipatch; engine supports computations using multiple patches.
+   The <type>UniformTag</type> indicates the patches all have the
+   same size.  Since patches may reside on different contexts, the
+   second template parameter is <type>Remote</type>.  Its
+   <type>Brick</type> template parameter specifies the engine for a
+   particular patch on a particular context.  Most distributed
+   programs use <type>MultiPatch&lt;UniformTag, Remote&lt;Brick&gt;
+   &gt;</type> or <type>MultiPatch&lt;UniformTag,
+   Remote&lt;CompressibleBrick&gt; &gt;</type> engines.</para>
+ 
+   <para>The computations for a distributed implementation are
+   exactly the same as for a sequential implementation.  The &pooma;
+   Toolkit and a message-passing library automatically perform all
+   computation.</para>
+ 
+   <para>Input and output for distributed programs is different than
+   for sequential programs.  Although the same instructions run on
+   each context, each context may have its own input and output
+   streams.  To avoid dealing with multiple input streams, we pass
+   the input via command-line arguments, which are replicated for
+   each context.  Using &inform; streams avoids having multiple
+   output streams print.  Any context can print to an &inform; stream
+   but only text sent to context&nbsp;0 is sent.  At the beginning of
+   the program, we create an &inform; object.  Throughout the rest of
+   the program, we use it instead of <varname>std::cout</varname> and
+   <varname>std::cerr</varname>.</para>
+ 
+   <para>The command to run the program is dependent on the run-time
+   system.  To use &mpi; with the Irix&nbsp;6.5 operating system, one
+   can use the <command>mpirun</command> command.  For example,
+   <statement>mpirun -np 4 Doof2d-Array-distributed -mpi 2 10
+   1000</statement> invokes the &mpi; run-time system with four
+   processors.  The <statement>-mpi</statement> option tells the
+   &pooma; executable <command>Doof2d-Array-distributed</command> to
+   use the &mpi; Library.  The remaining arguments specify the number
+   of processors, the number of averagings, and the array size.  The
+   first and last values are used for each dimension.  For example,
+   if three processors are specified, then the x-dimension will have
+   three processors and the y-dimension will have three processors,
+   totalling nine processors.  The command
+   <statement>Doof2d-Array-distributed -shmem -np 4 2 10
+   1000</statement> uses the &mm; Shared Memory Library
+   (<statement>-shmem</statement>) and four processors.  As for
+   &mpi;, the remaining command-line arguments are specified on a
+   per-dimension basis for the two-dimensional program.</para>
+  </section>
+ 
+ 
+  <section id="tutorial-field_data_parallel">
+   <title>Data-Parallel &field; Implementation</title>
+ 
+   <para>&pooma; &array;s support many scientific computations, but
+   many scientific computations require values distributed throughout
+   space, and &array;s have no spatial extent.  &pooma; &field;s,
+   supporting a superset of &array; functionality, model values
+   distributed throughout space.</para>
+ 
+   <para>A &field; consists of a set of cells distributed through
+   space.  Like an &array; cell, each &field; cell is addressed via
+   indices.  Unlike an &array; cell, each &field; cell can hold
+   multiple values.  Like &array;s, &field;s can be accessed via
+   data-parallel expressions and stencils and may be distributed
+   across processors.  Unlike &array; cells, &field; cells exist in a
+   multi-dimensional volume so, e.g., distances between cells and
+   normals to cells can be computed.</para>
+ 
+   <para>In this section, we implement the &doof2d; two-dimensional
+   diffusion simulation program using &field;s.  This simulation does
+   not require any &field;-specific features, but we chose to present
+   this program rather than one using &field;-specific features to
+   permit comparisons with the &array; versions, especially <xref
+   linkend="tutorial-array_parallel-doof2d"></xref>.</para>
+ 
+   <example id="tutorial-field_parallel-doof2d">
+    <title>Data-Parallel &field; Implementation of &doof2d;</title>
+    &doof2d-field-parallel;
+    <calloutlist>
+     <callout arearefs="tutorial-field_parallel-doof2d-header">
+      <para>To use &field;s, the <filename
+      class="headerfile">Pooma/Fields.h</filename> must be
+      included.</para>
+     </callout>
+     <callout arearefs="tutorial-field_parallel-doof2d-mesh">
+      <para>These statements specify the spacing and number of
+      &field; values.  First, a layout is explicitly.  Then, a mesh,
+      which specifies the spacing between cells, is created.  The
+      &field;'s centering specifies one cell-centered value per
+      cell.</para>
+     </callout>
+     <callout arearefs="tutorial-field_parallel-doof2d-field_creation">
+      <para>&field;'s first template parameter specifies the type of
+      mesh to use.  The other template parameters are similar to
+      &array;'s.  The constructor arguments specify the &field;'s
+      centering, its domain of cells, and a mesh specifying the
+      cells' spatial arrangement.</para>
+     </callout>
+     <callout arearefs="tutorial-field_parallel-doof2d-first_write">
+      <para>The computation for &field;s is the same as for &array;s
+      because this example does not use any &field;-specific
+      features.</para>
+     </callout>
+    </calloutlist>
+   </example>
+ 
+   <para>As mentioned above, the fundamental difference between
+   &array;s and &field;s is the latter has cells and meshes.  The
+   &field; declarations reflect this.  To declare a &field;, the
+   <filename class="headerfile">Pooma/Fields.h</filename> header file
+   must be included.  A &field;'s domain consists of a set of cells,
+   sometimes called positions when referring to &array;s.  As for
+   &array;s, a &field;'s domain and its layout must be specified.
+   Since the above program is designed for uniprocessor computation,
+   specifying the domain specifies the layout.  A &field;'s
+   <firstterm>mesh</firstterm> specifies its spatial extent.  For
+   example, one can ask the mesh for the distance between two cells
+   or for the normals to a particular cell.  Cells in a
+   <type>UniformRectilinearMesh</type> all have the same size and are
+   parallelepipeds.  To create the mesh, one specifies the layout,
+   the location of the spatial point corresponding to the lower, left
+   domain location, and the size of a particular cell.  Since this
+   program does not use mesh computations, our choices do not much
+   matter.  We specify the domain's lower, left corner is at spatial
+   location (0.0, 0.0) and each cell's width and height is&nbsp;1.
+   Thus, the middle of the cell at domain position (3,4) is (3.5,
+   4.5).</para>
+ 
+   <para>A &field; cell can contain one or more values although each
+   cell must have the same arrangement.  For this simulation, we
+   desire one value per cell so we place that position at the cell's
+   center, i.e., a cell centering.  The
+   <function>canonicalCentering</function> function returns such a
+   centering.  We defer discussion of the latter two arguments to
+   <xref linkend="sequential"></xref>.</para>
+ 
+   <para>A &field; declaration is analogous to an &array; declaration
+   but must also specify a centering and a mesh.  In <xref
+   linkend="tutorial-array_parallel-doof2d"></xref>, the &array;
+   declaration specifies the array's dimensionality, the value type,
+   the engine type, and a layout.  &field; declarations specify the
+   same values.  Its first template parameter specifies the mesh's
+   type, which includes an indication of its dimensionality.  The
+   second and third template parameters specify the value type and
+   the engine type.  Since a &field; has a centering and a mesh in
+   addition to a layout, those arguments are also necessary.</para>
+ 
+   <para>&field; operations are a superset of &array; operations so
+   the &doof2d; computations are the same as for <xref
+   linkend="tutorial-array_parallel-doof2d"></xref>.  &field;
+   accesses require parentheses, not square brackets, and accesses to
+   particular values should be preceded by calls to
+   <function>Pooma::blockAndEvaluate</function>.</para>
+ 
+   <para>To summarize, &field;s support multiple values per cell and
+   have spatial extent.  Thus, their declarations must specify a
+   centering and a mesh.  Otherwise, a &field; program is similar to
+   one with &array;s.</para>
+  </section>
+ 
+ 
+  <section id="tutorial-field_distributed">
+   <title>Distributed &field; Implementation</title>
+ 
+   <para>A &pooma; program using &field;s can execute on one or more
+   processors.  In <xref
+   linkend="tutorial-array_distributed"></xref>, we demonstrated how
+   to modify a uniprocessor stencil &array; implementation to run on
+   multiple processors.  In this section, we demonstrate that the
+   uniprocessor data-parallel &field; implementation of the previous
+   section can be converted.  Only the container declarations change;
+   the computations do not.  Since the changes are exactly analogous
+   to those in <xref linkend="tutorial-array_distributed"></xref>,
+   our exposition here will be shorter.</para>
+ 
+   <example id="tutorial-field_distributed-doof2d">
+    <title>Distributed Data-Parallel &field; Implementation of &doof2d;</title>
+    &doof2d-field-distributed;
+    <calloutlist>
+     <callout arearefs="tutorial-field_distributed-doof2d-io">
+      <para>Multiple copies of a distributed program may
+      simultaneously run, perhaps each having its own input and
+      output.  Thus, we use command-line arguments to pass input to
+      the program.  Using an &inform; stream ensures only one program
+      produces output.</para>
+     </callout>
+     <callout arearefs="tutorial-field_distributed-doof2d-layout">
+      <para>The <type>UniformGridPartition</type> declaration
+      specifies how an array's domain will be partition, of split,
+      into patches.  Guard layers are an optimization that can reduce
+      data communication between patches.  The
+      <type>UniformGridLayout</type> declaration applies the
+      partition to the given domain, distributing the resulting
+      patches among various processors.</para>
+     </callout>
+     <callout arearefs="tutorial-field_distributed-doof2d-mesh">
+      <para>The mesh and centering declarations are the same for
+      uniprocessor and multi-processor implementations.</para>
+     </callout>
+     <callout arearefs="tutorial-field_distributed-doof2d-remote">
+      <para>The <type>MultiPatch</type> &engine; distributes requests
+      for &array; values to the associated patch.  Since a patch may
+      associated with a different processor, its
+      <quote>remote</quote> engine has type
+      <type>Remote&lt;Brick&gt;</type>.  &pooma; automatically
+      distributes the patches among available memories and
+      processors.</para>
+     </callout>
+    </calloutlist>
+   </example>
+ 
+   <para>This program can be viewed as the combination of <xref
+   linkend="tutorial-field_parallel-doof2d"></xref> and the changes
+   to form the distributed stencil-based &array; program from the
+   uniprocessor stencil-based &array; program.
+ 
+   <itemizedlist>
+     <listitem>
+      <para>Distributed programs may have multiple processes, each
+      with its own input and output streams.  To pass input to these
+      processes, this programs uses command-line arguments, which are
+      replicated for each process.  An &inform; stream accepts data
+      from any context but prints only data from
+      context&nbsp;0.</para>
+     </listitem>
+     <listitem>
+      <para>A layout for a distributed program specifies a domain, a
+      partition, and a context mapper.  A &distributedtag; context
+      mapper tag indicates that pieces of the domain should be
+      distributed among patches, while a &replicatedtag; context
+      mapper tag indicates the entire domain should be replicated to
+      each patch.</para>
+     </listitem>
+     <listitem>
+      <para>A &multipatch; engine supports the use of multiple
+      patches, while a <type>remote</type> engine supports
+      computation distributed among various contexts.  Both are
+      usually necessary for distributed computation.</para>
+     </listitem>
+     <listitem>
+      <para>The computation for uniprocessor or distributed
+      implementations remains the same.  The &pooma; toolkit
+      automatically handles all communication necessary to ensure
+      up-to-date values are available when needed.</para>
+     </listitem>
+     <listitem>
+      <para>The command to invoke a distributed program is
+      system-dependent.  For example, the <statement>mpirun -np 4
+      Doof2d-Field-distributed -mpi 2 10 1000</statement> command
+      might use &mpi; communication.
+      <statement>Doof2d-Field-distributed -shmem -np 4 2 10
+      1000</statement> might use the &mm; Shared Memory Library.</para>
+     </listitem>
+   </itemizedlist>
+   </para>
+  </section>
+ <!-- FIXME: Do I need a chapter conclusion? -->
+ </chapter>
Index: figures/concepts.mp
===================================================================
RCS file: concepts.mp
diff -N concepts.mp
*** /dev/null	Fri Mar 23 21:37:44 2001
--- concepts.mp	Tue Dec 11 13:31:10 2001
***************
*** 0 ****
--- 1,207 ----
+ %% Oldham, Jeffrey D.
+ %% 2001Dec04
+ %% Pooma
+ 
+ %% Illustrations for Pooma Concepts
+ 
+ %% Assumes TEX=latex.
+ 
+ input boxes;
+ 
+ verbatimtex
+ \documentclass[10pt]{article}
+ \input{macros.ltx}
+ \begin{document}
+ etex
+ 
+ 
+ %% Container Declaration Concepts and Dependences
+ beginfig(111)
+   numeric unit; unit = 0.9cm;
+   numeric horizSpace; horizSpace = unit;
+   numeric vertSpace; vertSpace = unit;
+   
+   % Draw a line between two boxes.
+   vardef drawLine(expr start, stop) =
+     draw b[start].c -- b[stop].c cutbefore bpath b[start] cutafter bpath b[stop];
+   enddef;
+   
+   % Create boxes for the concepts.
+   boxit.b0(btex \type{TinyMatrix} etex);
+   boxit.b1(btex \type{Vector} etex);
+   boxit.b2(btex \type{DynamicArray} etex);
+   boxit.b3(btex \type{Array} etex);
+   boxit.b4(btex \type{Field} etex);
+   boxit.b5(btex engine etex);
+   boxit.b6(btex mesh etex);
+   boxit.b7(btex centering etex);
+   boxit.b8(btex layout etex);
+   boxit.b9(btex corner position etex);
+   boxit.b10(btex cell size etex);
+   boxit.b11(btex domain etex);
+   boxit.b12(btex partition etex);
+   boxit.b13(btex context mapper tag etex);
+   boxit.b14(btex guard layer etex);
+   boxit.b15(btex empty etex);
+   boxit.b16(btex distributed only etex); % dashed box surrounding distributed computation only elements
+   boxit.b17(btex \type{Tensor} etex);
+   boxit.b18(btex view box etex);
+   for t = 0 upto 15:
+     fixsize(b[t]);
+   endfor
+   fixsize(b17);
+ 
+   % Position the boxes.
+   b0.c = origin;
+   % horizontal positioning
+   for u = (0,3), (8,9), (11,12):
+     for t = xpart(u) upto ypart(u):
+       b[t+1].w - b[t].e = (horizSpace,0);
+     endfor
+   endfor
+   for u = (17,0), (5,15), (15,6), (6,7):
+     b[ypart(u)].w - b[xpart(u)].e = (horizSpace,0);
+   endfor
+   % vertical positioning
+   for u = (2,5), (6,9), (8,12), (12,14):
+     b[xpart(u)].s - b[ypart(u)].n = (0,vertSpace);
+   endfor
+   % distributed (dashed) box
+   b16.sw = b14.sw;
+   b16.ne = b13.ne;
+   % container view (dashed) box
+   b18.nw = b2.nw;
+   b18.se = b4.se;
+   
+   % Draw the boxes.
+   for t = 0 upto 18:
+     if unknown(b[t].c):
+       show t;
+       show b[t].c;
+     fi
+   endfor
+   for t = 0 upto 14:
+     drawunboxed(b[t]);
+   endfor
+   drawunboxed(b17);
+   
+   % Draw the dependences.
+   for t = 0 upto 4:		% lines to engine
+     drawLine(t,5);
+   endfor
+   drawLine(17,5);
+   for t = 2 upto 3:		% lines to layout
+     drawLine(t,8);
+   endfor
+   for t = 6 upto 7:		% lines from field
+     drawLine(4,t);
+   endfor
+   for t = 8 upto 10:		% lines from mesh
+     drawLine(6,t);
+   endfor
+   for t = 11 upto 13:		% lines from layout
+     drawLine(8,t);
+   endfor
+   drawLine(12,14);		% partition - guard layer
+ 
+   % Draw the dashed box around the distributed dependences.
+   draw bpath b16 dashed evenly;
+   label.llft(btex \textsl{multiprocessor computation only} etex, b16.se);
+ 
+   % Draw the dashed box around the container view box.
+   draw bpath b18 dashed evenly;
+   label.ulft(btex \textsl{support views} etex, b18.ne);
+ endfig;
+ 
+ 
+ %% Comparisons Between Mathematical Concept And Computational Implementation of Arrays and Fields
+ beginfig(101)
+   numeric unit; unit = 0.9cm;
+   numeric vertSpace; vertSpace = 2.6unit;   % vertical space between sections
+   numeric horizSpace; horizSpace = 8unit; % horizontal space between sections
+   
+   % Create and layout boxes for computational Array and Field implementations.
+   numeric interBoxSpace; interBoxSpace = unit; % space between boxes
+   numeric arrowAngle; arrowAngle = 20; % angle for arrow leaving index
+   path upperArrow[];
+   for t = 0 upto 1:
+     boxit.ia[t](btex index etex);
+     boxit.la[t](btex layout etex);
+     boxit.pa[t](btex \begin{tabular}{c} processors \\ memory \end{tabular} etex);
+     boxit.ea[t](btex engine etex);
+     boxit.va[t](btex value etex);
+ 
+     va[t].w - ea[t].e = (interBoxSpace,0);
+     ea[t].c - ia[t].c = 4(va[t].c - ea[t].c);
+     fixsize(ia[t],ea[t],va[t]);
+   endfor
+ 
+   % Create and layout text boxes.
+   boxit.l1(btex \strut mathematical concept etex);
+   boxit.l2(btex \strut computational implementation etex);
+   boxit.l3(btex \strut \type{Array}: etex);
+   boxit.l4(btex \strut $\mbox{index} \mapsto \mbox{value}$ etex);
+   boxit.l6(btex \strut \type{Field}: etex);
+   boxit.l7(btex \strut $\mbox{index} \mapsto \mbox{value}$ etex);
+   boxit.l9(btex \strut \type{Field}: etex);
+   boxit.l10(btex \strut $\mbox{indices} \mapsto \mbox{geometric value}$ etex);
+   fixsize(l1,l2,l3,l4,l6,l7,l9,l10);
+   
+   ypart(l1.c - l2.c) = 0;
+   xpart(l2.c - 0.5[ia[0].w,va[0].e]) = 0;
+   l1.w - l3.w = l4.w - l7.w = (0,vertSpace);
+   l4.w - l3.e = l7.nw - l6.ne = (0,0);
+   for t = 0 upto 1:
+     xpart(ia[t].w - l[3+3t].e) = 0.65horizSpace;
+     ypart(ia[t].w - l[3+3t].c) = 0;
+   endfor
+   xpart(l10.w - l7.w) = 0;
+   ypart(l10.w - ia[2].w) = 0;
+   ypart(l9.w - l10.w) = 0;
+   xpart(l9.w - l6.w) = 0;
+   
+   % Create and layout the mesh boxes.
+   boxit.ia[2](btex indices etex);
+   boxit.ea[2](btex mesh etex);
+   boxit.va[2](btex geometric value etex);
+   fixsize(ia[2],ea[2],va[2]);
+   ia[1].w - ia[2].w = 0.6(ia[0].w - ia[1].w);
+   ypart(va[2].w - ea[2].e) = ypart(ea[2].w - ia[2].e) = 0;
+   xpart(va[2].e - va[1].e) = 0;
+   xpart(ea[2].c - 0.5[ia[2].c,va[2].c]) = 0;
+ 
+   % Finish boxes on arrow for computational Array and Field implementations.
+   for t = 0 upto 1:
+     fixpos(ia[t],ea[t],va[t]);
+     upperArrow[t] = ia[t].c{dir arrowAngle} .. ea[t].c cutbefore bpath ia[t] cutafter bpath ea[t];
+     la[t].c = point 1/3 of upperArrow[t];
+     pa[t].c = point 2/3 of upperArrow[t];
+   endfor
+   
+   %% Draw the boxes.
+   % Draw the computational Array and Field implementations.
+   for t = 0 upto 1:
+ %  fixsize(ia,la,pa,ea,va); fixpos(ia,la,pa,ea,va);
+     draw ia[t].c{dir -arrowAngle} .. ea[t].c cutbefore bpath ia[t] cutafter bpath ea[t];
+     drawboxed(la[t],ea[t]);
+     drawunboxed(ia[t],pa[t],va[t]);
+     draw (subpath (0,1/3) of upperArrow[t]) cutafter bpath la[t];
+     drawarrow (subpath (1/3,2/3) of upperArrow[t]) cutbefore bpath la[t] cutafter bpath pa[t];
+     draw (subpath (2/3,1) of upperArrow[t]) cutbefore bpath pa[t] cutafter bpath ea[t];
+     drawarrow ea[t].e -- va[t].w;
+   endfor
+ 
+   % Draw the mesh boxes.
+   drawunboxed(ia[2],va[2]);
+   drawboxed(ea[2]);
+   draw ia[2].e -- ea[2].w cutbefore bpath ia[2] cutafter bpath ea[2];
+   drawarrow ea[2].e -- va[2].w cutbefore bpath ea[2] cutafter bpath va[2];
+ 
+   % Draw the text boxes.
+   for t = 1,2,3,4,6,7,9,10:
+     drawunboxed(l[t]);
+   endfor;
+   
+ endfig;
+ 
+ bye
Index: figures/distributed.mp
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/figures/distributed.mp,v
retrieving revision 1.1
diff -c -p -r1.1 distributed.mp
*** figures/distributed.mp	2001/12/04 00:07:00	1.1
--- figures/distributed.mp	2001/12/11 20:31:10
*************** input boxes;
*** 10,21 ****
  
  verbatimtex
  \documentclass[10pt]{article}
  \begin{document}
  etex
  
  %% Parts of Distributed Computation
  beginfig(101)
!   numeric unit; unit = 0.9cm;
  
    %% Create the Container Storage Partition subfigure.
    numeric arrayWidth; arrayWidth = 2; % as multiple of unit
--- 10,22 ----
  
  verbatimtex
  \documentclass[10pt]{article}
+ \input{macros.ltx}
  \begin{document}
  etex
  
  %% Parts of Distributed Computation
  beginfig(101)
!   numeric unit; unit = 0.8cm;
  
    %% Create the Container Storage Partition subfigure.
    numeric arrayWidth; arrayWidth = 2; % as multiple of unit
*************** beginfig(101)
*** 189,195 ****
  
    %% Draw the subfigure relations structures.
    drawunboxed(figurePlus,figureArrow);
!   label.rt(btex DistributedTag etex, figureArrow.e);
  endfig;
  
  bye
--- 190,196 ----
  
    %% Draw the subfigure relations structures.
    drawunboxed(figurePlus,figureArrow);
!   label.rt(btex \type{DistributedTag} etex, figureArrow.e);
  endfig;
  
  bye
Index: figures/doof2d.mp
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/figures/doof2d.mp,v
retrieving revision 1.1
diff -c -p -r1.1 doof2d.mp
*** figures/doof2d.mp	2001/12/04 00:07:00	1.1
--- figures/doof2d.mp	2001/12/11 20:31:10
***************
*** 8,13 ****
--- 8,14 ----
  
  verbatimtex
  \documentclass[10pt]{article}
+ \input{macros.ltx}
  \begin{document}
  etex
  
*************** beginfig(201)
*** 64,70 ****
    endfor
  
    % Label the grid.
!   labelGrid(btex Array \texttt{b}: Initial Configuration etex, nuCells, origin);
  endfig;
  
  
--- 65,71 ----
    endfor
  
    % Label the grid.
!   labelGrid(btex Array \varname{b}: Initial Configuration etex, nuCells, origin);
  endfig;
  
  
*************** beginfig(202)
*** 93,99 ****
    endfor
    
    % Label the grid.
!   labelGrid(btex Array \texttt{a}: After the first averaging etex, nuCells, origin);
  endfig;
  
  
--- 94,100 ----
    endfor
    
    % Label the grid.
!   labelGrid(btex Array \varname{a}: After the first averaging etex, nuCells, origin);
  endfig;
  
  
*************** beginfig(203)
*** 133,139 ****
    endfor
    
    % Label the grid.
!   labelGrid(btex Array \texttt{b}: After the second averaging etex, nuCells, origin);
  endfig;
  
  
--- 134,140 ----
    endfor
    
    % Label the grid.
!   labelGrid(btex Array \varname{b}: After the second averaging etex, nuCells, origin);
  endfig;
  
  
Index: figures/macros.ltx
===================================================================
RCS file: macros.ltx
diff -N macros.ltx
*** /dev/null	Fri Mar 23 21:37:44 2001
--- macros.ltx	Tue Dec 11 13:31:10 2001
***************
*** 0 ****
--- 1,14 ----
+ %% Oldham, Jeffrey D.
+ %% 2001Dec05
+ %% Pooma
+ 
+ %% Macros for Figures
+ 
+ %% Consistency between these macros and the DocBook/Jade output is desired.
+ 
+ \newcommand{\type}[1]{\texttt{#1}}%
+ 	% Produce a C++ (or other programming language) type.
+ 	% Requires:	1. the type's name.
+ \newcommand{\varname}[1]{\texttt{#1}}%
+ 	% Produce a C++ (or other programming language) variable.
+ 	% Requires:	1. the variable's name.
Index: programs/Doof2d-Array-distributed-annotated.patch
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/programs/Doof2d-Array-distributed-annotated.patch,v
retrieving revision 1.1
diff -c -p -r1.1 Doof2d-Array-distributed-annotated.patch
*** programs/Doof2d-Array-distributed-annotated.patch	2001/12/04 00:07:00	1.1
--- programs/Doof2d-Array-distributed-annotated.patch	2001/12/11 20:31:10
***************
*** 1,8 ****
! *** Doof2d-Array-distributed.cpp	Wed Nov 28 07:46:56 2001
! --- Doof2d-Array-distributed-annotated.cpp	Wed Nov 28 07:53:31 2001
  ***************
! *** 1,4 ****
! ! #include <iostream>		// has std::cout, ...
  ! #include <stdlib.h>		// has EXIT_SUCCESS
    #include "Pooma/Arrays.h"	// has Pooma's Array
    
--- 1,7 ----
! *** Doof2d-Array-distributed.cpp	Wed Dec  5 14:04:36 2001
! --- Doof2d-Array-distributed-annotated.cpp	Wed Dec  5 14:07:56 2001
  ***************
! *** 1,3 ****
  ! #include <stdlib.h>		// has EXIT_SUCCESS
    #include "Pooma/Arrays.h"	// has Pooma's Array
    
***************
*** 13,19 ****
    #include "Pooma/Arrays.h"	// has Pooma's Array
    
  ***************
! *** 15,19 ****
      // (i,j).  The "C" template parameter permits use of this stencil
      // operator with both Arrays and Fields.
  !   template <class C>
--- 12,18 ----
    #include "Pooma/Arrays.h"	// has Pooma's Array
    
  ***************
! *** 14,18 ****
      // (i,j).  The "C" template parameter permits use of this stencil
      // operator with both Arrays and Fields.
  !   template <class C>
***************
*** 26,65 ****
      inline
      typename C::Element_t
  ***************
! *** 40,52 ****
!     Pooma::initialize(argc,argv);
!     
! !   // Ask the user for the number of processors.
      long nuProcessors;
! !   std::cout << "Please enter the number of processors: ";
! !   std::cin >> nuProcessors;
    
!     // Ask the user for the number of averagings.
      long nuAveragings, nuIterations;
! !   std::cout << "Please enter the number of averagings: ";
! !   std::cin >> nuAveragings;
      nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
    
! --- 41,53 ----
!     Pooma::initialize(argc,argv);
!     
! !   // Ask the user for the number of processors.  <co id="tutorial-array_distributed-doof2d-nuprocessors"></co>
      long nuProcessors;
! !   std::cout &lt;&lt; "Please enter the number of processors: ";
! !   std::cin &gt;&gt; nuProcessors;
    
!     // Ask the user for the number of averagings.
      long nuAveragings, nuIterations;
! !   std::cout &lt;&lt; "Please enter the number of averagings: ";
! !   std::cin &gt;&gt; nuAveragings;
      nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
    
  ***************
! *** 54,67 ****
      // the grid.
      long n;
! !   std::cout << "Please enter the array size: ";
! !   std::cin >> n;
    
      // Specify the arrays' domains [0,n) x [0,n).
  !   Interval<1> N(0, n-1);
--- 25,91 ----
      inline
      typename C::Element_t
  ***************
! *** 42,46 ****
!     // canot use standard input and output.  Instead we use command-line
!     // arguments, which are replicated, for input, and we use an Inform
! !   // stream for output.
!     Inform output;
!   
! --- 44,48 ----
!     // canot use standard input and output.  Instead we use command-line
!     // arguments, which are replicated, for input, and we use an Inform
! !   // stream for output.  <co id="tutorial-array_distributed-doof2d-io"></co>
!     Inform output;
!   
! ***************
! *** 48,52 ****
!     if (argc != 4) {
!       // Incorrect number of command-line arguments.
! !     output << argv[0] << ": number-of-processors number-of-averagings number-of-values" << std::endl;
!       return EXIT_FAILURE;
!     }
! --- 50,54 ----
!     if (argc != 4) {
!       // Incorrect number of command-line arguments.
! !     output &lt;&lt; argv[0] &lt;&lt; ": number-of-processors number-of-averagings number-of-values" &lt;&lt; std::endl;
!       return EXIT_FAILURE;
!     }
! ***************
! *** 55,63 ****
!     // Determine the number of processors.
      long nuProcessors;
! !   nuProcessors = strtol(argv[1], &tail, 0);
    
!     // Determine the number of averagings.
      long nuAveragings, nuIterations;
! !   nuAveragings = strtol(argv[2], &tail, 0);
      nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
    
! --- 57,65 ----
!     // Determine the number of processors.
      long nuProcessors;
! !   nuProcessors = strtol(argv[1], &amp;tail, 0);
    
!     // Determine the number of averagings.
      long nuAveragings, nuIterations;
! !   nuAveragings = strtol(argv[2], &amp;tail, 0);
      nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
    
  ***************
! *** 65,69 ****
!     // the grid.
!     long n;
! !   n = strtol(argv[3], &tail, 0);
!     // The dimension must be a multiple of the number of processors
!     // since we are using a UniformGridLayout.
! --- 67,71 ----
      // the grid.
      long n;
! !   n = strtol(argv[3], &amp;tail, 0);
!     // The dimension must be a multiple of the number of processors
!     // since we are using a UniformGridLayout.
! ***************
! *** 71,80 ****
    
      // Specify the arrays' domains [0,n) x [0,n).
  !   Interval<1> N(0, n-1);
***************
*** 70,80 ****
  !   Interval<2> interiorDomain(I,I);
    
      // Create the distributed arrays.
! --- 55,68 ----
!     // the grid.
!     long n;
! !   std::cout &lt;&lt; "Please enter the array size: ";
! !   std::cin &gt;&gt; n;
    
      // Specify the arrays' domains [0,n) x [0,n).
  !   Interval&lt;1&gt; N(0, n-1);
--- 96,102 ----
  !   Interval<2> interiorDomain(I,I);
    
      // Create the distributed arrays.
! --- 73,82 ----
    
      // Specify the arrays' domains [0,n) x [0,n).
  !   Interval&lt;1&gt; N(0, n-1);
***************
*** 86,92 ****
    
      // Create the distributed arrays.
  ***************
! *** 70,85 ****
      // dimension.  Guard layers optimize communication between patches.
      // Internal guards surround each patch.  External guards surround
  !   // the entire array domain.
--- 108,114 ----
    
      // Create the distributed arrays.
  ***************
! *** 83,98 ****
      // dimension.  Guard layers optimize communication between patches.
      // Internal guards surround each patch.  External guards surround
  !   // the entire array domain.
***************
*** 103,109 ****
  !   Array<2, double, MultiPatch<UniformTag, Remote<Brick> > > b(layout);
    
      // Set up the initial conditions.
! --- 71,86 ----
      // dimension.  Guard layers optimize communication between patches.
      // Internal guards surround each patch.  External guards surround
  !   // the entire array domain.  <co id="tutorial-array_distributed-doof2d-layout"></co>
--- 125,131 ----
  !   Array<2, double, MultiPatch<UniformTag, Remote<Brick> > > b(layout);
    
      // Set up the initial conditions.
! --- 85,100 ----
      // dimension.  Guard layers optimize communication between patches.
      // Internal guards surround each patch.  External guards surround
  !   // the entire array domain.  <co id="tutorial-array_distributed-doof2d-layout"></co>
***************
*** 121,127 ****
    
      // Set up the initial conditions.
  ***************
! *** 89,97 ****
    
      // Create the stencil performing the computation.
  !   Stencil<DoofNinePt> stencil;
--- 143,149 ----
    
      // Set up the initial conditions.
  ***************
! *** 104,112 ****
    
      // Create the stencil performing the computation.
  !   Stencil<DoofNinePt> stencil;
***************
*** 131,137 ****
  !     // Read from b.  Write to a.
        a(interiorDomain) = stencil(b, interiorDomain);
    
! --- 90,98 ----
    
      // Create the stencil performing the computation.
  !   Stencil&lt;DoofNinePt&gt; stencil;
--- 153,159 ----
  !     // Read from b.  Write to a.
        a(interiorDomain) = stencil(b, interiorDomain);
    
! --- 106,114 ----
    
      // Create the stencil performing the computation.
  !   Stencil&lt;DoofNinePt&gt; stencil;
***************
*** 142,162 ****
        a(interiorDomain) = stencil(b, interiorDomain);
    
  ***************
! *** 102,106 ****
      // Print out the final central value.
      Pooma::blockAndEvaluate();	// Ensure all computation has finished.
! !   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
    
      // The arrays are automatically deallocated.
! --- 103,107 ----
      // Print out the final central value.
      Pooma::blockAndEvaluate();	// Ensure all computation has finished.
! !   std::cout &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
    
      // The arrays are automatically deallocated.
  ***************
! *** 110,111 ****
! --- 111,113 ----
      return EXIT_SUCCESS;
    }
  + </programlisting>
--- 164,184 ----
        a(interiorDomain) = stencil(b, interiorDomain);
    
  ***************
! *** 117,121 ****
      // Print out the final central value.
      Pooma::blockAndEvaluate();	// Ensure all computation has finished.
! !   output << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
    
      // The arrays are automatically deallocated.
! --- 119,123 ----
      // Print out the final central value.
      Pooma::blockAndEvaluate();	// Ensure all computation has finished.
! !   output &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
    
      // The arrays are automatically deallocated.
  ***************
! *** 125,126 ****
! --- 127,129 ----
      return EXIT_SUCCESS;
    }
  + </programlisting>
Index: programs/Doof2d-Array-element-annotated.patch
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/programs/Doof2d-Array-element-annotated.patch,v
retrieving revision 1.1
diff -c -p -r1.1 Doof2d-Array-element-annotated.patch
*** programs/Doof2d-Array-element-annotated.patch	2001/12/04 00:07:00	1.1
--- programs/Doof2d-Array-element-annotated.patch	2001/12/11 20:31:10
***************
*** 1,5 ****
! *** Doof2d-Array-element.cpp	Tue Nov 27 11:04:04 2001
! --- Doof2d-Array-element-annotated.cpp	Tue Nov 27 12:06:32 2001
  ***************
  *** 1,5 ****
  ! #include <iostream>		// has std::cout, ...
--- 1,5 ----
! *** Doof2d-Array-element.cpp	Tue Dec  4 12:02:10 2001
! --- Doof2d-Array-element-annotated.cpp	Tue Dec  4 12:24:25 2001
  ***************
  *** 1,5 ****
  ! #include <iostream>		// has std::cout, ...
***************
*** 58,65 ****
    
      // Set up the initial conditions.
  !   // All grid values should be zero except for the central value.
!     a = b = 0.0;
!     b(n/2,n/2) = 1000.0;
  --- 20,38 ----
      // the grid.
      long n;
--- 58,65 ----
    
      // Set up the initial conditions.
  !   // All grid values should be zero except for the central value.
!     for (int j = 1; j < n-1; j++)
!       for (int i = 1; i < n-1; i++)
  --- 20,38 ----
      // the grid.
      long n;
***************
*** 78,87 ****
    
      // Set up the initial conditions.
  !   // All grid values should be zero except for the central value.  <co id="tutorial-array_elementwise-doof2d-initialization"></co>
!     a = b = 0.0;
!     b(n/2,n/2) = 1000.0;
  ***************
! *** 41,49 ****
    
      // Perform the simulation.
  !   for (int k = 0; k < nuIterations; ++k) {
--- 78,87 ----
    
      // Set up the initial conditions.
  !   // All grid values should be zero except for the central value.  <co id="tutorial-array_elementwise-doof2d-initialization"></co>
!     for (int j = 1; j < n-1; j++)
!       for (int i = 1; i < n-1; i++)
  ***************
! *** 43,51 ****
    
      // Perform the simulation.
  !   for (int k = 0; k < nuIterations; ++k) {
***************
*** 91,97 ****
  !         a(i,j) = weight *
              (b(i+1,j+1) + b(i+1,j  ) + b(i+1,j-1) +
               b(i  ,j+1) + b(i  ,j  ) + b(i  ,j-1) +
! --- 42,50 ----
    
      // Perform the simulation.
  !   for (int k = 0; k &lt; nuIterations; ++k) {
--- 91,97 ----
  !         a(i,j) = weight *
              (b(i+1,j+1) + b(i+1,j  ) + b(i+1,j-1) +
               b(i  ,j+1) + b(i  ,j  ) + b(i  ,j-1) +
! --- 44,52 ----
    
      // Perform the simulation.
  !   for (int k = 0; k &lt; nuIterations; ++k) {
***************
*** 102,115 ****
              (b(i+1,j+1) + b(i+1,j  ) + b(i+1,j-1) +
               b(i  ,j+1) + b(i  ,j  ) + b(i  ,j-1) +
  ***************
! *** 51,56 ****
    
        // Read from a.  Write to b.
  !     for (int j = 1; j < n-1; j++)
  !       for (int i = 1; i < n-1; i++)
            b(i,j) = weight *
              (a(i+1,j+1) + a(i+1,j  ) + a(i+1,j-1) +
! --- 52,57 ----
    
        // Read from a.  Write to b.
  !     for (int j = 1; j &lt; n-1; j++)
--- 102,115 ----
              (b(i+1,j+1) + b(i+1,j  ) + b(i+1,j-1) +
               b(i  ,j+1) + b(i  ,j  ) + b(i  ,j-1) +
  ***************
! *** 53,58 ****
    
        // Read from a.  Write to b.
  !     for (int j = 1; j < n-1; j++)
  !       for (int i = 1; i < n-1; i++)
            b(i,j) = weight *
              (a(i+1,j+1) + a(i+1,j  ) + a(i+1,j-1) +
! --- 54,59 ----
    
        // Read from a.  Write to b.
  !     for (int j = 1; j &lt; n-1; j++)
***************
*** 117,126 ****
            b(i,j) = weight *
              (a(i+1,j+1) + a(i+1,j  ) + a(i+1,j-1) +
  ***************
! *** 60,70 ****
    
      // Print out the final central value.
- !   Pooma::blockAndEvaluate();	// Ensure all computation has finished.
  !   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
    
  !   // The arrays are automatically deallocated.
--- 117,125 ----
            b(i,j) = weight *
              (a(i+1,j+1) + a(i+1,j  ) + a(i+1,j-1) +
  ***************
! *** 62,71 ****
    
      // Print out the final central value.
  !   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
    
  !   // The arrays are automatically deallocated.
***************
*** 129,138 ****
      Pooma::finalize();
      return EXIT_SUCCESS;
    }
! --- 61,72 ----
    
      // Print out the final central value.
! !   Pooma::blockAndEvaluate();	// Ensure all computation has finished.  <co id="tutorial-array_elementwise-doof2d-blockAndEvaluate"></co>
  !   std::cout &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
    
  !   // The arrays are automatically deallocated.  <co id="tutorial-array_elementwise-doof2d-deallocation"></co>
--- 128,137 ----
      Pooma::finalize();
      return EXIT_SUCCESS;
    }
! --- 63,74 ----
    
      // Print out the final central value.
! !   Pooma::blockAndEvaluate();	// Ensure all computation has finished.
  !   std::cout &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
    
  !   // The arrays are automatically deallocated.  <co id="tutorial-array_elementwise-doof2d-deallocation"></co>
Index: programs/Doof2d-Array-parallel-annotated.patch
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/programs/Doof2d-Array-parallel-annotated.patch,v
retrieving revision 1.1
diff -c -p -r1.1 Doof2d-Array-parallel-annotated.patch
*** programs/Doof2d-Array-parallel-annotated.patch	2001/12/04 00:07:00	1.1
--- programs/Doof2d-Array-parallel-annotated.patch	2001/12/11 20:31:10
***************
*** 1,5 ****
! *** Doof2d-Array-parallel.cpp	Tue Nov 27 13:00:09 2001
! --- Doof2d-Array-parallel-annotated.cpp	Tue Nov 27 14:07:07 2001
  ***************
  *** 1,4 ****
  ! #include <iostream>		// has std::cout, ...
--- 1,5 ----
! *** Doof2d-Array-parallel.cpp	Tue Dec  4 11:49:43 2001
! --- Doof2d-Array-parallel-annotated.cpp	Tue Dec  4 12:24:36 2001
  ***************
  *** 1,4 ****
  ! #include <iostream>		// has std::cout, ...
***************
*** 28,34 ****
      nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
    
  ***************
! *** 19,38 ****
      // the grid.
      long n;
  !   std::cout << "Please enter the array size: ";
--- 28,34 ----
      nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
    
  ***************
! *** 19,43 ****
      // the grid.
      long n;
  !   std::cout << "Please enter the array size: ";
***************
*** 49,55 ****
  !   Array<2, double, Brick> b(vertDomain);
    
      // Set up the initial conditions.
! --- 20,39 ----
      // the grid.
      long n;
  !   std::cout &lt;&lt; "Please enter the array size: ";
--- 49,60 ----
  !   Array<2, double, Brick> b(vertDomain);
    
      // Set up the initial conditions.
!     // All grid values should be zero except for the central value.
!     a = b = 0.0;
! !   // Ensure all data-parallel computation finishes before accessing a value.
!     Pooma::blockAndEvaluate();
!     b(n/2,n/2) = 1000.0;
! --- 20,44 ----
      // the grid.
      long n;
  !   std::cout &lt;&lt; "Please enter the array size: ";
***************
*** 70,84 ****
  !   Array&lt;2, double, Brick&gt; b(vertDomain);
    
      // Set up the initial conditions.
  ***************
! *** 45,50 ****
    
      // Perform the simulation.
  !   for (int k = 0; k < nuIterations; ++k) {
  !     // Read from b.  Write to a.
        a(I,J) = weight *
          (b(I+1,J+1) + b(I+1,J  ) + b(I+1,J-1) +
! --- 46,51 ----
    
      // Perform the simulation.
  !   for (int k = 0; k &lt; nuIterations; ++k) {
--- 75,94 ----
  !   Array&lt;2, double, Brick&gt; b(vertDomain);
    
      // Set up the initial conditions.
+     // All grid values should be zero except for the central value.
+     a = b = 0.0;
+ !   // Ensure all data-parallel computation finishes before accessing a value.  <co id="tutorial-array_parallel-doof2d-blockAndEvaluate"></co>
+     Pooma::blockAndEvaluate();
+     b(n/2,n/2) = 1000.0;
  ***************
! *** 47,52 ****
    
      // Perform the simulation.
  !   for (int k = 0; k < nuIterations; ++k) {
  !     // Read from b.  Write to a.
        a(I,J) = weight *
          (b(I+1,J+1) + b(I+1,J  ) + b(I+1,J-1) +
! --- 48,53 ----
    
      // Perform the simulation.
  !   for (int k = 0; k &lt; nuIterations; ++k) {
***************
*** 86,106 ****
        a(I,J) = weight *
          (b(I+1,J+1) + b(I+1,J  ) + b(I+1,J-1) +
  ***************
! *** 61,65 ****
      // Print out the final central value.
      Pooma::blockAndEvaluate();	// Ensure all computation has finished.
  !   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
    
      // The arrays are automatically deallocated.
! --- 62,66 ----
      // Print out the final central value.
      Pooma::blockAndEvaluate();	// Ensure all computation has finished.
  !   std::cout &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
    
      // The arrays are automatically deallocated.
  ***************
! *** 69,70 ****
! --- 70,72 ----
      return EXIT_SUCCESS;
    }
  + </programlisting>
--- 96,116 ----
        a(I,J) = weight *
          (b(I+1,J+1) + b(I+1,J  ) + b(I+1,J-1) +
  ***************
! *** 63,67 ****
      // Print out the final central value.
      Pooma::blockAndEvaluate();	// Ensure all computation has finished.
  !   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
    
      // The arrays are automatically deallocated.
! --- 64,68 ----
      // Print out the final central value.
      Pooma::blockAndEvaluate();	// Ensure all computation has finished.
  !   std::cout &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
    
      // The arrays are automatically deallocated.
  ***************
! *** 71,72 ****
! --- 72,74 ----
      return EXIT_SUCCESS;
    }
  + </programlisting>
Index: programs/Doof2d-Array-stencil-annotated.patch
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/programs/Doof2d-Array-stencil-annotated.patch,v
retrieving revision 1.1
diff -c -p -r1.1 Doof2d-Array-stencil-annotated.patch
*** programs/Doof2d-Array-stencil-annotated.patch	2001/12/04 00:07:00	1.1
--- programs/Doof2d-Array-stencil-annotated.patch	2001/12/11 20:31:10
***************
*** 1,5 ****
! *** Doof2d-Array-stencil.cpp	Tue Nov 27 17:23:41 2001
! --- Doof2d-Array-stencil-annotated.cpp	Tue Nov 27 17:36:13 2001
  ***************
  *** 1,9 ****
  ! #include <iostream>		// has std::cout, ...
--- 1,5 ----
! *** Doof2d-Array-stencil.cpp	Tue Dec  4 11:49:39 2001
! --- Doof2d-Array-stencil-annotated.cpp	Tue Dec  4 12:26:46 2001
  ***************
  *** 1,9 ****
  ! #include <iostream>		// has std::cout, ...
***************
*** 109,115 ****
    
      // Set up the initial conditions.
  ***************
! *** 71,80 ****
      b(n/2,n/2) = 1000.0;
    
  !   // Create the stencil performing the computation.
--- 109,115 ----
    
      // Set up the initial conditions.
  ***************
! *** 73,82 ****
      b(n/2,n/2) = 1000.0;
    
  !   // Create the stencil performing the computation.
***************
*** 120,126 ****
  !     // Read from b.  Write to a.
        a(interiorDomain) = stencil(b, interiorDomain);
    
! --- 72,81 ----
      b(n/2,n/2) = 1000.0;
    
  !   // Create the stencil performing the computation.  <co id="tutorial-array_stencil-doof2d-stencil_creation"></co>
--- 120,126 ----
  !     // Read from b.  Write to a.
        a(interiorDomain) = stencil(b, interiorDomain);
    
! --- 74,83 ----
      b(n/2,n/2) = 1000.0;
    
  !   // Create the stencil performing the computation.  <co id="tutorial-array_stencil-doof2d-stencil_creation"></co>
***************
*** 132,152 ****
        a(interiorDomain) = stencil(b, interiorDomain);
    
  ***************
! *** 85,89 ****
      // Print out the final central value.
      Pooma::blockAndEvaluate();	// Ensure all computation has finished.
  !   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
    
      // The arrays are automatically deallocated.
! --- 86,90 ----
      // Print out the final central value.
      Pooma::blockAndEvaluate();	// Ensure all computation has finished.
  !   std::cout &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
    
      // The arrays are automatically deallocated.
  ***************
! *** 93,94 ****
! --- 94,96 ----
      return EXIT_SUCCESS;
    }
  + </programlisting>
--- 132,152 ----
        a(interiorDomain) = stencil(b, interiorDomain);
    
  ***************
! *** 87,91 ****
      // Print out the final central value.
      Pooma::blockAndEvaluate();	// Ensure all computation has finished.
  !   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
    
      // The arrays are automatically deallocated.
! --- 88,92 ----
      // Print out the final central value.
      Pooma::blockAndEvaluate();	// Ensure all computation has finished.
  !   std::cout &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
    
      // The arrays are automatically deallocated.
  ***************
! *** 95,96 ****
! --- 96,98 ----
      return EXIT_SUCCESS;
    }
  + </programlisting>
Index: programs/Doof2d-C-element-annotated.patch
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/programs/Doof2d-C-element-annotated.patch,v
retrieving revision 1.1
diff -c -p -r1.1 Doof2d-C-element-annotated.patch
*** programs/Doof2d-C-element-annotated.patch	2001/12/04 00:07:00	1.1
--- programs/Doof2d-C-element-annotated.patch	2001/12/11 20:31:10
***************
*** 75,81 ****
          a[i][j] = b[i][j] = 0.0;
      b[n/2][n/2] = 1000.0;
    
! !   // In the average, weight element with this value.
      const double weight = 1.0/9.0;
    
      // Perform the simulation.
--- 75,81 ----
          a[i][j] = b[i][j] = 0.0;
      b[n/2][n/2] = 1000.0;
    
! !   // In the average, weight elements with this value.
      const double weight = 1.0/9.0;
    
      // Perform the simulation.
***************
*** 94,100 ****
          a[i][j] = b[i][j] = 0.0;
      b[n/2][n/2] = 1000.0;
    
! !   // In the average, weight element with this value.  <co id="tutorial-hand_coded-doof2d-constants"></co>
      const double weight = 1.0/9.0;
    
      // Perform the simulation.
--- 94,100 ----
          a[i][j] = b[i][j] = 0.0;
      b[n/2][n/2] = 1000.0;
    
! !   // In the average, weight elements with this value.  <co id="tutorial-hand_coded-doof2d-constants"></co>
      const double weight = 1.0/9.0;
    
      // Perform the simulation.
Index: programs/Doof2d-Field-distributed-annotated.patch
===================================================================
RCS file: Doof2d-Field-distributed-annotated.patch
diff -N Doof2d-Field-distributed-annotated.patch
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-Field-distributed-annotated.patch	Tue Dec 11 13:31:10 2001
***************
*** 0 ****
--- 1,176 ----
+ *** Doof2d-Field-distributed.cpp	Wed Dec  5 14:05:10 2001
+ --- Doof2d-Field-distributed-annotated.cpp	Wed Dec  5 14:41:24 2001
+ ***************
+ *** 1,3 ****
+ ! #include <stdlib.h>		// has EXIT_SUCCESS
+   #include "Pooma/Fields.h"	// has Pooma's Field
+   
+ --- 1,4 ----
+ ! <programlisting id="tutorial-field_distributed-doof2d-program" linenumbering="numbered" format="linespecific">
+ ! #include &lt;stdlib.h&gt;		// has EXIT_SUCCESS
+   #include "Pooma/Fields.h"	// has Pooma's Field
+   
+ ***************
+ *** 12,16 ****
+     // canot use standard input and output.  Instead we use command-line
+     // arguments, which are replicated, for input, and we use an Inform
+ !   // stream for output.
+     Inform output;
+   
+ --- 13,17 ----
+     // canot use standard input and output.  Instead we use command-line
+     // arguments, which are replicated, for input, and we use an Inform
+ !   // stream for output.  <co id="tutorial-field_distributed-doof2d-io"></co>
+     Inform output;
+   
+ ***************
+ *** 18,22 ****
+     if (argc != 4) {
+       // Incorrect number of command-line arguments.
+ !     output << argv[0] << ": number-of-processors number-of-averagings number-of-values" << std::endl;
+       return EXIT_FAILURE;
+     }
+ --- 19,23 ----
+     if (argc != 4) {
+       // Incorrect number of command-line arguments.
+ !     output &lt;&lt; argv[0] &lt;&lt; ": number-of-processors number-of-averagings number-of-values" &lt;&lt; std::endl;
+       return EXIT_FAILURE;
+     }
+ ***************
+ *** 25,33 ****
+     // Determine the number of processors.
+     long nuProcessors;
+ !   nuProcessors = strtol(argv[1], &tail, 0);
+   
+     // Determine the number of averagings.
+     long nuAveragings, nuIterations;
+ !   nuAveragings = strtol(argv[2], &tail, 0);
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ --- 26,34 ----
+     // Determine the number of processors.
+     long nuProcessors;
+ !   nuProcessors = strtol(argv[1], &amp;tail, 0);
+   
+     // Determine the number of averagings.
+     long nuAveragings, nuIterations;
+ !   nuAveragings = strtol(argv[2], &amp;tail, 0);
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ ***************
+ *** 35,39 ****
+     // the grid.
+     long n;
+ !   n = strtol(argv[3], &tail, 0);
+     // The dimension must be a multiple of the number of processors
+     // since we are using a UniformGridLayout.
+ --- 36,40 ----
+     // the grid.
+     long n;
+ !   n = strtol(argv[3], &amp;tail, 0);
+     // The dimension must be a multiple of the number of processors
+     // since we are using a UniformGridLayout.
+ ***************
+ *** 41,50 ****
+   
+     // Specify the fields' domains [0,n) x [0,n).
+ !   Interval<1> N(0, n-1);
+ !   Interval<2> vertDomain(N, N);
+   
+     // Set up interior domains [1,n-1) x [1,n-1) for computation.
+ !   Interval<1> I(1,n-2);
+ !   Interval<1> J(1,n-2);
+   
+     // Partition the fields' domains uniformly, i.e., each patch has the
+ --- 42,51 ----
+   
+     // Specify the fields' domains [0,n) x [0,n).
+ !   Interval&lt;1&gt; N(0, n-1);
+ !   Interval&lt;2&gt; vertDomain(N, N);
+   
+     // Set up interior domains [1,n-1) x [1,n-1) for computation.
+ !   Interval&lt;1&gt; I(1,n-2);
+ !   Interval&lt;1&gt; J(1,n-2);
+   
+     // Partition the fields' domains uniformly, i.e., each patch has the
+ ***************
+ *** 52,74 ****
+     // dimension.  Guard layers optimize communication between patches.
+     // Internal guards surround each patch.  External guards surround
+ !   // the entire field domain.
+ !   UniformGridPartition<2> partition(Loc<2>(nuProcessors, nuProcessors),
+ ! 				    GuardLayers<2>(1),  // internal
+ ! 				    GuardLayers<2>(0)); // external
+ !   UniformGridLayout<2> layout(vertDomain, partition, DistributedTag());
+   
+     // Specify the fields' mesh, i.e., its spatial extent, and its
+ !   // centering type.
+ !   UniformRectilinearMesh<2> mesh(layout, Vector<2>(0.0), Vector<2>(1.0, 1.0));
+ !   Centering<2> cell = canonicalCentering<2>(CellType, Continuous, AllDim);
+   
+     // The template parameters indicate a mesh and a 'double'
+     // element type.  MultiPatch indicates multiple computation patches,
+     // i.e., distributed computation.  The UniformTag indicates the
+ !   // patches should have the same size.  Each patch has Brick type.
+ !   Field<UniformRectilinearMesh<2>, double, MultiPatch<UniformTag,
+ !     Remote<Brick> > > a(cell, layout, mesh);
+ !   Field<UniformRectilinearMesh<2>, double, MultiPatch<UniformTag,
+ !     Remote<Brick> > > b(cell, layout, mesh);
+   
+     // Set up the initial conditions.
+ --- 53,75 ----
+     // dimension.  Guard layers optimize communication between patches.
+     // Internal guards surround each patch.  External guards surround
+ !   // the entire field domain.  <co id="tutorial-field_distributed-doof2d-layout"></co>
+ !   UniformGridPartition&lt;2&gt; partition(Loc&lt;2&gt;(nuProcessors, nuProcessors),
+ ! 				    GuardLayers&lt;2&gt;(1),  // internal
+ ! 				    GuardLayers&lt;2&gt;(0)); // external
+ !   UniformGridLayout&lt;2&gt; layout(vertDomain, partition, DistributedTag());
+   
+     // Specify the fields' mesh, i.e., its spatial extent, and its
+ !   // centering type.  <co id="tutorial-field_distributed-doof2d-mesh"></co>
+ !   UniformRectilinearMesh&lt;2&gt; mesh(layout, Vector&lt;2&gt;(0.0), Vector&lt;2&gt;(1.0, 1.0));
+ !   Centering&lt;2&gt; cell = canonicalCentering&lt;2&gt;(CellType, Continuous, AllDim);
+   
+     // The template parameters indicate a mesh and a 'double'
+     // element type.  MultiPatch indicates multiple computation patches,
+     // i.e., distributed computation.  The UniformTag indicates the
+ !   // patches should have the same size.  Each patch has Brick type.  <co id="tutorial-field_distributed-doof2d-remote"></co>
+ !   Field&lt;UniformRectilinearMesh&lt;2&gt;, double, MultiPatch&lt;UniformTag,
+ !     Remote&lt;Brick&gt; &gt; &gt; a(cell, layout, mesh);
+ !   Field&lt;UniformRectilinearMesh&lt;2&gt;, double, MultiPatch&lt;UniformTag,
+ !     Remote&lt;Brick&gt; &gt; &gt; b(cell, layout, mesh);
+   
+     // Set up the initial conditions.
+ ***************
+ *** 83,87 ****
+   
+     // Perform the simulation.
+ !   for (int k = 0; k < nuIterations; ++k) {
+       // Read from b.  Write to a.
+       a(I,J) = weight *
+ --- 84,88 ----
+   
+     // Perform the simulation.
+ !   for (int k = 0; k &lt; nuIterations; ++k) {
+       // Read from b.  Write to a.
+       a(I,J) = weight *
+ ***************
+ *** 99,103 ****
+     // Print out the final central value.
+     Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+ !   output << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
+   
+     // The fields are automatically deallocated.
+ --- 100,104 ----
+     // Print out the final central value.
+     Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+ !   output &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
+   
+     // The fields are automatically deallocated.
+ ***************
+ *** 107,108 ****
+ --- 108,110 ----
+     return EXIT_SUCCESS;
+   }
+ + </programlisting>
Index: programs/Doof2d-Field-parallel-annotated.patch
===================================================================
RCS file: Doof2d-Field-parallel-annotated.patch
diff -N Doof2d-Field-parallel-annotated.patch
*** /dev/null	Fri Mar 23 21:37:44 2001
--- Doof2d-Field-parallel-annotated.patch	Tue Dec 11 13:31:10 2001
***************
*** 0 ****
--- 1,120 ----
+ *** Doof2d-Field-parallel.cpp	Tue Dec  4 10:01:28 2001
+ --- Doof2d-Field-parallel-annotated.cpp	Tue Dec  4 11:04:26 2001
+ ***************
+ *** 1,5 ****
+ ! #include <iostream>		// has std::cout, ...
+ ! #include <stdlib.h>		// has EXIT_SUCCESS
+ ! #include "Pooma/Fields.h"	// has Pooma's Field
+   
+   // Doof2d: Pooma Fields, data-parallel implementation
+ --- 1,6 ----
+ ! <programlisting id="tutorial-field_parallel-doof2d-program" linenumbering="numbered" format="linespecific">
+ ! #include &lt;iostream&gt;		// has std::cout, ...
+ ! #include &lt;stdlib.h&gt;		// has EXIT_SUCCESS
+ ! #include "Pooma/Fields.h"	// has Pooma's Field  <co id="tutorial-field_parallel-doof2d-header"></co>
+   
+   // Doof2d: Pooma Fields, data-parallel implementation
+ ***************
+ *** 12,17 ****
+     // Ask the user for the number of averagings.
+     long nuAveragings, nuIterations;
+ !   std::cout << "Please enter the number of averagings: ";
+ !   std::cin >> nuAveragings;
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ --- 13,18 ----
+     // Ask the user for the number of averagings.
+     long nuAveragings, nuIterations;
+ !   std::cout &lt;&lt; "Please enter the number of averagings: ";
+ !   std::cin &gt;&gt; nuAveragings;
+     nuIterations = (nuAveragings+1)/2; // Each iteration performs two averagings.
+   
+ ***************
+ *** 19,44 ****
+     // the grid.
+     long n;
+ !   std::cout << "Please enter the field size: ";
+ !   std::cin >> n;
+   
+     // Specify the fields' domains [0,n) x [0,n).
+ !   Interval<1> N(0, n-1);
+ !   Interval<2> vertDomain(N, N);
+   
+     // Set up interior domains [1,n-1) x [1,n-1) for computation.
+ !   Interval<1> I(1,n-2);
+ !   Interval<1> J(1,n-2);
+   
+     // Specify the fields' mesh, i.e., its spatial extent, and its
+ !   // centering type.
+ !   DomainLayout<2> layout(vertDomain);
+ !   UniformRectilinearMesh<2> mesh(layout, Vector<2>(0.0), Vector<2>(1.0, 1.0));
+ !   Centering<2> cell = canonicalCentering<2>(CellType, Continuous, AllDim);
+   
+     // Create the fields.
+     // The template parameters indicate a mesh, a 'double' element
+ !   // type, and ordinary 'Brick' storage.
+ !   Field<UniformRectilinearMesh<2>, double, Brick> a(cell, layout, mesh);
+ !   Field<UniformRectilinearMesh<2>, double, Brick> b(cell, layout, mesh);
+   
+     // Set up the initial conditions.
+ --- 20,45 ----
+     // the grid.
+     long n;
+ !   std::cout &lt;&lt; "Please enter the field size: ";
+ !   std::cin &gt;&gt; n;
+   
+     // Specify the fields' domains [0,n) x [0,n).
+ !   Interval&lt;1&gt; N(0, n-1);
+ !   Interval&lt;2&gt; vertDomain(N, N);
+   
+     // Set up interior domains [1,n-1) x [1,n-1) for computation.
+ !   Interval&lt;1&gt; I(1,n-2);
+ !   Interval&lt;1&gt; J(1,n-2);
+   
+     // Specify the fields' mesh, i.e., its spatial extent, and its
+ !   // centering type.  <co id="tutorial-field_parallel-doof2d-mesh"></co>
+ !   DomainLayout&lt;2&gt; layout(vertDomain);
+ !   UniformRectilinearMesh&lt;2&gt; mesh(layout, Vector&lt;2&gt;(0.0), Vector&lt;2&gt;(1.0, 1.0));
+ !   Centering&lt;2&gt; cell = canonicalCentering&lt;2&gt;(CellType, Continuous, AllDim);
+   
+     // Create the fields.
+     // The template parameters indicate a mesh, a 'double' element
+ !   // type, and ordinary 'Brick' storage.  <co id="tutorial-field_parallel-doof2d-field_creation"></co>
+ !   Field&lt;UniformRectilinearMesh&lt;2&gt;, double, Brick&gt; a(cell, layout, mesh);
+ !   Field&lt;UniformRectilinearMesh&lt;2&gt;, double, Brick&gt; b(cell, layout, mesh);
+   
+     // Set up the initial conditions.
+ ***************
+ *** 51,56 ****
+   
+     // Perform the simulation.
+ !   for (int k = 0; k < nuIterations; ++k) {
+ !     // Read from b.  Write to a.
+       a(I,J) = weight *
+         (b(I+1,J+1) + b(I+1,J  ) + b(I+1,J-1) +
+ --- 52,57 ----
+   
+     // Perform the simulation.
+ !   for (int k = 0; k &lt; nuIterations; ++k) {
+ !     // Read from b.  Write to a.  <co id="tutorial-field_parallel-doof2d-first_write"></co>
+       a(I,J) = weight *
+         (b(I+1,J+1) + b(I+1,J  ) + b(I+1,J-1) +
+ ***************
+ *** 67,71 ****
+     // Print out the final central value.
+     Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+ !   std::cout << (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) << std::endl;
+   
+     // The fields are automatically deallocated.
+ --- 68,72 ----
+     // Print out the final central value.
+     Pooma::blockAndEvaluate();	// Ensure all computation has finished.
+ !   std::cout &lt;&lt; (nuAveragings % 2 ? a(n/2,n/2) : b(n/2,n/2)) &lt;&lt; std::endl;
+   
+     // The fields are automatically deallocated.
+ ***************
+ *** 75,76 ****
+ --- 76,78 ----
+     return EXIT_SUCCESS;
+   }
+ + </programlisting>

From oldham at codesourcery.com  Tue Dec 11 21:44:14 2001
From: oldham at codesourcery.com (Jeffrey Oldham)
Date: Tue, 11 Dec 2001 13:44:14 -0800
Subject: Patch: Fix Typos in Comments
Message-ID: <20011211134414.B29015@codesourcery.com>

This patch fixes some typographical errors in Pooma source code
comments.

2001-Dec-11  Jeffrey D. Oldham  <oldham at codesourcery.com>

        * Field/Mesh/UniformRectilinearMesh.h
        (UniformRectilinearMeshData): Remove extraneous conjunctive in
        comment.
        (UniformRectilinearMeshData::~UniformRectilinearMeshData): Fix
        typo in introdutory comment.
        * Partition/UniformGridPartition.h
        (UniformGridPartition<Dim>::partition): Fix typos in comment.
        * Utilities/Pooma.cmpl.cpp: Fix typo in comment.

Not tested since the changes are only to comments.
Applied to      mainline
Approved by     Mark Mitchell

Thanks,
Jeffrey D. Oldham
oldham at codesourcery.com
-------------- next part --------------
Index: Field/Mesh/UniformRectilinearMesh.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Field/Mesh/UniformRectilinearMesh.h,v
retrieving revision 1.3
diff -c -p -r1.3 UniformRectilinearMesh.h
*** Field/Mesh/UniformRectilinearMesh.h	2001/12/03 19:38:33	1.3
--- Field/Mesh/UniformRectilinearMesh.h	2001/12/11 19:27:44
*************** public:
*** 63,69 ****
    // Constructors.
  
    // We provide a default constructor that creates the object with empty
!   // domains and. To be useful, this object must be replaced by another 
    // version via assignment.
    
    UniformRectilinearMeshData()
--- 63,69 ----
    // Constructors.
  
    // We provide a default constructor that creates the object with empty
!   // domains. To be useful, this object must be replaced by another 
    // version via assignment.
    
    UniformRectilinearMeshData()
*************** public:
*** 150,156 ****
  
    //---------------------------------------------------------------------------
    // Empty destructor is fine. Note, however, that NoMeshData does not have
!   // a virtual destructor. We must be carefult to delete these puppies as
    // UniformRectilinearMeshData.
  
    ~UniformRectilinearMeshData() { }
--- 150,156 ----
  
    //---------------------------------------------------------------------------
    // Empty destructor is fine. Note, however, that NoMeshData does not have
!   // a virtual destructor. We must be careful to delete these puppies as
    // UniformRectilinearMeshData.
  
    ~UniformRectilinearMeshData() { }
Index: Partition/UniformGridPartition.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Partition/UniformGridPartition.h,v
retrieving revision 1.27
diff -c -p -r1.27 UniformGridPartition.h
*** Partition/UniformGridPartition.h	2001/12/03 19:38:34	1.27
--- Partition/UniformGridPartition.h	2001/12/11 19:27:44
*************** public:
*** 375,382 ****
  	      {
  		iguards = internalGuards_m;
              
! 		// Check if we're at an edge, and if so use the
! 		// external specfication for that edge.
                  
  		for (int d = 0; d < Dim; ++d)
  		  {
--- 375,382 ----
  	      {
  		iguards = internalGuards_m;
              
! 		// Check if we're at an edge, and, if so, use the
! 		// external specification for that edge.
                  
  		for (int d = 0; d < Dim; ++d)
  		  {
Index: Pooma/Pooma.cmpl.cpp
===================================================================
RCS file: /home/pooma/Repository/r2/src/Pooma/Pooma.cmpl.cpp,v
retrieving revision 1.37
diff -c -p -r1.37 Pooma.cmpl.cpp
*** Pooma/Pooma.cmpl.cpp	2001/04/11 21:39:28	1.37
--- Pooma/Pooma.cmpl.cpp	2001/12/11 19:27:45
*************** void lockThreads(bool on)
*** 855,861 ****
  }
  
  //-----------------------------------------------------------------------------
! // Return whether threads hsould be locked to processors.
  //-----------------------------------------------------------------------------
  
  bool blockingExpressions()
--- 855,861 ----
  }
  
  //-----------------------------------------------------------------------------
! // Return whether threads should be locked to processors.
  //-----------------------------------------------------------------------------
  
  bool blockingExpressions()

From oldham at codesourcery.com  Thu Dec 13 05:05:31 2001
From: oldham at codesourcery.com (Jeffrey Oldham)
Date: Wed, 12 Dec 2001 21:05:31 -0800
Subject: Manual Patch: Finish Concepts Chapter
Message-ID: <20011212210531.A13125@codesourcery.com>

2001-Dec-12  Jeffrey D. Oldham  <oldham at codesourcery.com>

	This patch mainly finishes the first draft of the Pooma concepts
	chapter.
	
	* concepts.xml: New file containing the chapter describing the
	Pooma concepts.  Some of this material was moved out of
	manual.xml.  The "Computation Modes" and "Computation Environment"
	material is new.
	* glossary.xml: Added entries corresponding to concepts added to
	concepts.xml.
	* manual.xml: Concepts chapter moved into concepts.xml.  Unused
	material moved to "Writing Sequential Programs" chapter.
	* figures/concepts.mp: Changed "geometric value" to "spatial
	value".
	
Applied to	mainline.

Thanks,
Jeffrey D. Oldham
oldham at codesourcery.com
-------------- next part --------------
Index: concepts.xml
===================================================================
RCS file: concepts.xml
diff -N concepts.xml
*** /dev/null	Fri Mar 23 21:37:44 2001
--- concepts.xml	Wed Dec 12 21:02:05 2001
***************
*** 0 ****
--- 1,630 ----
+ <chapter id="concepts">
+  <title>Overview of &pooma; Concepts</title>
+ 
+  <para>FIXME: How does multi-threaded computation fit into the
+  model?</para>
+ 
+  <para>In the previous chapter, we presented several different
+  implementations of the &doof2d; simulation program.  The
+  implementations illustrate the various containers, computation
+  modes, and computation environments that &pooma; supports.  In this
+  chapter, we describe the concepts associated with each of these
+  three categories.  Specific details needed by programmers are
+  deferred to later chapters.</para>
+ 
+  <table frame="none" colsep="0" rowsep="0" tocentry="1"
+    orient="port" pgwide="0" id="concepts-table">
+    <title>&pooma; Implementation Concepts</title>
+    <tgroup cols="3" align="center">
+     <thead>
+      <row>
+       <entry>Container</entry>
+ 	<entry>Computation Modes</entry>
+ 	<entry>Computation Environment</entry>
+      </row>
+     </thead>
+     <tbody>
+      <row>
+ 	<entry>&array;</entry>
+ 	<entry>element-wise</entry>
+ 	<entry>sequential</entry>
+      </row>
+      <row>
+ 	<entry>&dynamicarray;</entry>
+ 	<entry>data-parallel</entry>
+ 	<entry>distributed</entry>
+      </row>
+      <row>
+ 	<entry>&field;</entry>
+ 	<entry>stencil-based</entry>
+ 	<entry></entry>
+      </row>
+      <row>
+ 	<entry>&tensor;</entry>
+ 	<entry>relational</entry>
+ 	<entry></entry>
+      </row>
+      <row>
+ 	<entry>&matrix;</entry>
+ 	<entry></entry>
+ 	<entry></entry>
+      </row>
+      <row>
+ 	<entry>&vector;</entry>
+ 	<entry></entry>
+ 	<entry></entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+ 
+  <para>The most important &pooma; concepts can be grouped into three
+  separate categories:
+  <variablelist>
+   <varlistentry>
+     <term>container</term>
+     <listitem>
+      <para>data structure holding one or more values and addressed
+      by indices</para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>computation modes</term>
+     <listitem>
+      <para>styles of expressing computations</para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>computation environment</term>
+     <listitem>
+      <para>description of resources for computing, e.g., single
+      processor or multi-processor</para>
+     </listitem>
+    </varlistentry>
+   </variablelist>
+   See <xref linkend="concepts-table"></xref>.  Many &pooma; programs
+   select one possibility from each column.  For example, <xref
+   linkend="tutorial-array_stencil-doof2d"></xref> used a &array;
+   container and stencils for sequential computation, while <xref
+   linkend="tutorial-field_distributed-doof2d"></xref> used a &field;
+   container and data-parallel statements with distributed
+   computation.  A program may use multiple containers and various
+   computation modes, but the computation environment either has
+   distributed processors or not.</para>
+ 
+  <para>In the rest of this chapter, we explore these three
+  categories.  First, we describe &pooma; containers, illustrating
+  the purposes of each, and explaining the concepts needed to declare
+  them.  Then, we describe the different computation modes and
+  finally distributed computation concepts.</para>
+ 
+ 
+  <section id="concepts-containers">
+   <title>&pooma; Containers</title>
+ 
+   <para>Most &pooma; programs use <firstterm>containers</firstterm>
+   to store groups of values.  &pooma; containers are objects that
+   store other objects.  They control allocation and deallocation of
+   and access to these objects.  They are a generalization of &c;
+   arrays, but &pooma; containers are first-class objects so they can
+   be used directly in expressions.  They are similar to &cc;
+   containers such as <type>vector</type>, <type>list</type>, and
+   <type>stack</type>.  See <xref
+   linkend="concepts-containers-table"></xref> for a summary of the
+   containers.</para>
+ 
+   <para>This chapter describes many concepts, not all of which are
+   needed to begin programming with the &pooma; Toolkit.  Below we
+   introduce the different categories of concepts.  After that, we
+   introduce the different &pooma;'s containers and describe how to
+   choose the appropriate one for a particular task.  <xref
+   linkend="concepts-sequential_containers-declarations-dependences"></xref>
+   indicates which concepts must be understood when declaring a
+   particular container.  All of these concepts are described in
+   <xref
+   linkend="concepts-containers-sequential_declarations"></xref> and
+   <xref
+   linkend="concepts-containers-distributed_declarations"></xref>.
+   Use this figure to decide which concepts in the former are
+   relevant.  Reading the latter section is necessary only if
+   computing using multiple processors.  The programs in the previous
+   chapter illustrate many of these concepts.</para>
+ 
+   <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	   orient="port" pgwide="0" id="concepts-containers-table">
+    <title>&pooma; Container Summary</title>
+    <tgroup cols="2" align="left">
+     <tbody>
+      <row>
+ 	<entry><glossterm
+ 	linkend="glossary-array">&array;</glossterm></entry>
+ 	<entry>container mapping indices to values and that may be
+ 	used in expressions</entry>
+      </row>
+      <row>
+ 	<entry><glossterm
+ 	linkend="glossary-dynamicarray">&dynamicarray;</glossterm></entry>
+ 	<entry>one-dimensional &array; whose <glossterm
+ 	linkend="glossary-domain">domain</glossterm> can be dynamically
+ 	resized</entry>
+      </row>
+      <row>
+ 	<entry><glossterm linkend="glossary-field">&field;</glossterm></entry>
+ 	<entry>container mapping indices to one or more values and
+ 	residing in multi-dimensional space</entry>
+      </row>
+      <row>
+ 	<entry><glossterm
+ 	linkend="glossary-tensor">&tensor;</glossterm></entry>
+ 	<entry>multi-dimensional mathematical tensor</entry>
+      </row>
+      <row>
+ 	<entry><glossterm
+ 	linkend="glossary-matrix">&matrix;</glossterm></entry>
+ 	<entry>two-dimensional mathematical matrix</entry>
+      </row>
+      <row>
+ 	<entry><glossterm
+ 	linkend="glossary-vector">&vector;</glossterm></entry>
+ 	<entry>multi-dimensional mathematical vector</entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+ 
+ <!-- FIXME: Want firstterm around array. -->
+ 
+   <para>A &pooma; <glossterm
+   linkend="glossary-array">array;</glossterm>, generalizing a &c;
+   array, maps indices to values.  Given a index or position in an
+   &array;'s domain, it returns the associated value, either by
+   returning a stored value or by computing it.  The use of indices,
+   which are usually ordered tuples, permits constant-time access
+   although computing a particular value may require significant
+   time.  In addition to the functionality provided by &c; arrays,
+   the &array; class automatically handles memory allocation and
+   deallocation, supports a wider variety of assignments, and can be
+   used in expressions.  For example, the addition of two arrays can
+   be assigned to an array and the product of a scalar element and an
+   array is permissible.</para>
+ 
+ <!-- FIXME: Want firstterm around dynamicarray. -->
+ 
+   <para>A &pooma; <glossterm
+   linkend="glossary-dynamicarray">&dynamicarray;</glossterm> extends
+   &array; capabilities to support a dynamically-changing domain but
+   is restricted to only one dimension.  When the &dynamicarray; is
+   resized, its values are preserved.</para>
+ 
+ <!-- FIXME: Want firstterm around field. -->
+ 
+   <para>A &pooma; <glossterm>&field;</glossterm> is an &array; with
+   spatial extent.  Each domain consists of <glossterm
+   linkend="glossary-cell"><firstterm>cell</firstterm></glossterm>s
+   in one-, two-, or three-dimensional space.  Although indexed
+   similarly to &array;s, each cell may contain multiple values and
+   multiple materials.  A &field;'s <glossterm
+   linkend="glossary-mesh">mesh</glossterm> stores its spatial
+   characteristics and can map yield, e.g., a point contained in a
+   cell, the distance between two cells, and a cell's normals.  A
+   &field; should be used whenever geometric or spatial computations
+   are needed, multiple values per index are desired, or a
+   computation involves more than one material.</para>
+ 
+ <!-- FIXME: Want firstterm around tensor. -->
+ 
+   <para>A <glossterm linkend="glossary-tensor">&tensor;</glossterm>
+   implements a multi-dimensional mathematical tensor.  Since it is a
+   first-class type, it can be used in expressions such as
+   adding two &tensor;s.</para>
+ 
+ <!-- FIXME: Want firstterm around matrix. -->
+ 
+   <para>A <glossterm linkend="glossary-matrix">&matrix;</glossterm>
+   implements a two-dimensional mathematical matrix.  Since it is a
+   first-class type, it can be used in expressions such as
+   multiplying matrices and assignments to matrices.</para>
+ 
+ <!-- FIXME: Want firstterm around vector. -->
+ 
+   <para>A <glossterm linkend="glossary-vector">&vector;</glossterm>
+   implements a multi-dimensional mathematical vector, which is an
+   ordered tuple of components.  Since it is a first-class type, it
+   can be used in expressions such as adding two &vector;s and
+   multiplying a &matrix; and a &vector;.</para>
+ 
+   <para>The data of an &array;, &dynamicarray;, or &field; can be
+   viewed using more than one container by taking a view.  A
+   <glossterm
+   linkend="glossary-view"><firstterm>view</firstterm></glossterm> of
+   an existing container&nbsp;&container; is a container whose domain
+   is a subset of&nbsp;&container;.  The subset can equal the
+   original domain.  A view acts like a reference in that changing
+   any of the view's values also changes the original container's and
+   vice versa.  While users sometimes explicitly create views, they
+   are perhaps more frequently created as temporaries in expressions.
+   For example, if <varname>A</varname> is an &array; and
+   <varname>I</varname> is a domain, <statement>A(I) -
+   A(I-1)</statement> forms the difference between adjacent
+   values.</para>
+ 
+ 
+   <section id="concepts-containers-choosing">
+    <title>Choosing a Container</title>
+ 
+    <para>The two most commonly used &pooma; containers are &array;s
+    and &field;s.  <xref
+    linkend="concepts-containers-choice_table"></xref> contains a
+    decision tree describing how to choose an appropriate
+    container.</para>
+ 
+   <table frame="none" colsep="0" rowsep="0" tocentry="1"
+ 	   orient="port" pgwide="0" id="concepts-containers-choice_table">
+    <title>Choosing a &pooma; Container</title>
+    <tgroup cols="2" align="left">
+     <tbody>
+      <row>
+ 	<entry>If modeling mathematical entries,</entry>
+ 	<entry>use a &vector;, &matrix;, or &tensor;.</entry>
+      </row>
+      <row>
+ 	<entry>If indices and values reside in multi-dimensional space
+ 	&space;,</entry>
+ 	<entry>use a &field;.</entry>
+      </row>
+      <row>
+ 	<entry>If there are multiple values per index,</entry>
+ 	<entry>use a &field;.</entry>
+      </row>
+      <row>
+ 	<entry>If there are multiple materials participating in the same computation,</entry>
+ 	<entry>use a &field;.</entry>
+      </row>
+      <row>
+ 	<entry>If the domain's size dynamically changes and is one-dimensional,</entry>
+ 	<entry>use a &dynamicarray;.</entry>
+      </row>
+      <row>
+ 	<entry>Otherwise</entry>
+ 	<entry>use an &array;.</entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+ 
+   </section><!-- choosing among containers -->
+ 
+ 
+   <section id="concepts-containers-sequential_declarations">
+    <title>Declaring Sequential Containers</title>
+ 
+    <figure float="1" id="concepts-sequential_containers-declarations-dependences">
+     <title>Concepts For Declaring Containers</title>
+     <mediaobject>
+      <imageobject>
+ 	<imagedata fileref="figures/concepts.111" format="EPS" align="center"></imagedata>
+      </imageobject>
+      <textobject>
+ 	<phrase>concepts involved in declaring containers</phrase>
+      </textobject>
+     </mediaobject>
+    </figure>
+ 
+    <para>In the previous sections, we introduced the &pooma;
+    containers and described how to choose one appropriate for a
+    given task.  In this section, we describe the concepts involved
+    in declaring them.  Concepts specific to distributed computation
+    are described in the next section.</para>
+ 
+    <para><xref
+    linkend="concepts-sequential_containers-declarations-dependences"></xref>
+    illustrates the containers and the concepts involved in their
+    declarations.  The containers are listed in the top row.  Lines
+    connect these containers to the components necessary for their
+    declarations.  For example, an &array; declaration requires an
+    engine and a layout.  These, in turn, depend on other &pooma;
+    concepts.  Declarations necessary only for distributed, or
+    multiprocessor, computation are surrounded by dashed lines.  You
+    can use these dependences to indicate the concepts needed for a
+    particular container.</para>
+ 
+    <para>An <glossterm
+    linkend="glossary-engine"><firstterm>engine</firstterm></glossterm>
+    stores and, if necessary, computes a container's values.  A
+    container has one or more engines.  The separation of a container
+    and its storage permits optimizing a program's space
+    requirements.  For example, a container returning the same value
+    for all indices can use a constant engine, which need only store
+    one value for the entire domain.  A &compressiblebrick; engine
+    reduces its space requirements to a constant whenever all its
+    values are the same.  The separation also permits taking <link
+    linkend="glossary-view">view</link>s of containers without
+    copying storage.</para>
+ 
+    <figure float="1" id="concepts-containers-declarations-computational_implementation">
+     <title>&array; and &field; Mathematical and Computational Concepts</title>
+     <mediaobject>
+      <imageobject>
+ 	<imagedata fileref="figures/concepts.101" format="EPS" align="center"></imagedata>
+      </imageobject>
+      <textobject>
+ 	<phrase>maps from indices to values</phrase>
+      </textobject>
+     </mediaobject>
+    </figure>
+ 
+    <para>A <glossterm
+    linkend="glossary-layout"><firstterm>layout</firstterm></glossterm>
+    maps <link linkend="glossary-domain">domain</link> indices to the
+    processors and computer memory used by a container's engines.
+    See <xref
+    linkend="concepts-containers-declarations-computational_implementation"></xref>.
+    A computer computes a container's values using a processor and
+    memory.  The layout specifies the processor(s) and memory to use
+    for each particular index.  A container's layout for a
+    uniprocessor implementation consists of its domain, the
+    processor, and its memory.  For a multi-processor implementation,
+    the layout maps portions of the domain to (possibly different)
+    processors and memory.</para>
+ 
+    <para>A &field;'s <glossterm
+    linkend="glossary-mesh"><firstterm>mesh</firstterm></glossterm>
+    maps <link linkend="glossary-domain">domain</link> indices to
+    spatial values in &space; such as distance between cells, edge
+    lengths, and normals to cells.  In other words, it provides a
+    &field;'s spatial extent.  See also <xref
+    linkend="concepts-containers-declarations-computational_implementation"></xref>.
+    Different mesh types may support different spatial
+    values.</para>
+ 
+    <para>A mesh's <glossterm
+    linkend="glossary-corner_position"><firstterm>corner
+    position</firstterm></glossterm> specifies the <link
+    linkend="glossary-point">point</link> in &space; corresponding to
+    the lower, left corner of its <link
+    linkend="glossary-domain">domain</link>.  Combining this, the
+    domain, and the cell size fully specifies the mesh's map from
+    indices to &space;.</para>
+ 
+    <para>A mesh's <glossterm
+    linkend="glossary-cell_size"><firstterm>cell
+    size</firstterm></glossterm> specifies the spatial dimensions of
+    a &field; <link linkend="glossary-cell">cell</link>, e.g., its
+    width, height, and depth, in&nbsp;&space;.  Combining this, the
+    domain, and the corner position fully specifies the mesh's map
+    from indices to &space;.</para>
+ 
+    <para>A <glossterm
+    linkend="glossary-domain"><firstterm>domain</firstterm></glossterm>
+    is a set of points on which a container can define values.  An
+    <glossterm
+    linkend="glossary-interval"><firstterm>interval</firstterm></glossterm>
+    consists of all integral points between two values.  It is
+    frequently represented using mathematical interval notation [a,b]
+    even though it contains only the integral points, e.g., a, a+1,
+    a+2, &hellip;, b.  The concept is generalized to multiple
+    dimensions by forming tensor product of intervals, i.e., all the
+    integral tuples in an &n;-dimensional space.  For example, the
+    two-dimensional containers in the previous chapter are defined on
+    a two-dimensional domain with the both dimensions' spanning the
+    interval [0,n).  A <glossterm
+    linkend="glossary-stride"><firstterm>stride</firstterm></glossterm>
+    is a subset of an interval consisting of regularly-spaced
+    points.  A <glossterm
+    linkend="glossary-range"><firstterm>range</firstterm></glossterm>
+    is a subset of an interval formed by the tensor product of strides.
+    A <glossterm
+    linkend="glossary-region"><firstterm>region</firstterm></glossterm>
+    represents a continuous &n;-dimensional domain.</para>
+   </section>
+ 
+ 
+   <section id="concepts-containers-distributed_declarations">
+    <title>Declaring Distributed Containers</title>
+ 
+    <para>In the previous section, we introduced the concepts
+    important when declaring containers for use on uniprocessor
+    computers.  When using multi-processor computers, we augment
+    these concepts with those for distributed computation.  Reading
+    this section is important only for running the same program on
+    multiple processors.  Many of these concepts were introduced in
+    <xref linkend="tutorial-array_distributed"></xref> and <xref
+    linkend="tutorial-field_distributed"></xref>.  <xref
+    linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
+    illustrates the &pooma; distributed computation model.  In this
+    section, we concentrate on the concepts necessary to declare a
+    distributed container.</para>
+ 
+    <para>As we noted in <xref
+    linkend="tutorial-array_distributed"></xref>, a &pooma;
+    programmer must specify how each container's domain should be
+    distributed among the available processors and memory spaces.
+    Using this information, the Toolkit automatically distributes the
+    data among the available processors and handles any required
+    communication among them.  The three concepts necessary for
+    declaring distributed containers are a partition, a guard layer,
+    and a context mapper tag.</para>
+ 
+    <para>A <glossterm
+    linkend="glossary-partition"><firstterm>partition</firstterm></glossterm>
+    specified how to divide a container's domain into distributed
+    pieces.  For example, the partition illustrated in <xref
+    linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
+    would divide a two-dimensional domain into three equally-sized
+    pieces along the x-dimension and two equally-sized pieces along
+    the y-dimension.  Partitions can be independent of the size of
+    container's domain.  The example partition will work on any
+    domain as long as the size of its x-dimension is a multiple of
+    three.  A domain is separated into disjoint <link
+    linkend="glossary-patch">patch</link>es.</para>
+ 
+    <para>A <glossterm
+    linkend="glossary-guard_layer"><firstterm>guard
+    layer</firstterm></glossterm> is <quote>extra</quote> domain
+    surrounding each patch.  This region has read-only values.  An
+    <glossterm
+    linkend="glossary-external_guard_layer"><firstterm>external guard
+    layer</firstterm></glossterm> specifies values surrounding the
+    domain.  Its presence eases computation along the domain's edges
+    by permitting the same computations as for more internal
+    computations.  An <glossterm
+    linkend="glossary-internal_guard_layer"><firstterm>internal guard
+    layer</firstterm></glossterm> duplicates values from adjacent
+    patches so communication with adjacent patches need not occur
+    during a patch's computation.  The use of guard layers is an
+    optimization; using external guard layers eases programming and
+    using internal guard layers reduces communication among
+    processors.  Their use is not required.</para>
+ 
+    <para>A <glossterm
+    linkend="glossary-context_mapper"><firstterm>context
+    mapper</firstterm></glossterm> indicates how a container's
+    patches are mapped to processors and shared memory.  For example,
+    the &distributedtag; indicates that the patches should be
+    distributed among the processors so each patch occurs once in the
+    entire computation.  The &replicatedtag; indicates that the
+    patches should be replicated among the processors so each
+    processing unit has its own copy of all the patches.  While it
+    could be wasteful to have different processors perform the same
+    computation, replicating a container can reduce possibly more
+    expensive communication costs.</para>
+   </section>
+  </section>
+ 
+ 
+  <section id="concepts-computation_modes">
+   <title>Computation Modes</title>
+ 
+   <para>&pooma; computations can be expressed using a variety of
+   modes.  Most of &pooma; computations involve &array; or &field;
+   containers, but how their values are accessed and the associated
+   algorithms using them varies.  Element-wise computation involves
+   explicitly accessing values.  A data-parallel computation uses
+   expressions to represent larger subsets of a container's values.
+   Stencil-based computations write a computation as repeatedly
+   applying a local computation to each element of an array.  A
+   relation among containers establishes a dependency between them so
+   the values of one container are updated whenever any other's
+   values change.  A program may use any or all of these styles,
+   described below.</para>
+ 
+   <para><glossterm
+   linkend="glossary-element_wise"><firstterm>Element-wise</firstterm></glossterm>
+   computation accesses individual container values through explicit
+   notation.  For example, values in a two-dimensional
+   container&nbsp;&container; might be referenced as
+   <statement>&container(3,4)</statement> or
+   <statement>&container(i,j+1)</statement>.  This is the usual
+   notation for languages without objects such as &c;.</para>
+ 
+   <para><glossterm
+   linkend="glossary-data_parallel"><firstterm>Data-parallel</firstterm></glossterm>
+   computation uses expressions to access subsets of a container's
+   values.  For example, in <xref
+   linkend="tutorial-array_parallel-doof2d"></xref>,
+   <statement>a(I,J)</statement> represents the subset of &array;
+   <varname>a</varname>'s values with coordinates in the domain
+   specified by the one-dimensional &interval;s <varname>I</varname>
+   and <varname>J</varname>.  Using data-parallel expressions
+   frequently eliminates the need for writing explicit loops in
+   code.</para>
+ 
+   <para>A <glossterm
+   linkend="glossary-stencil"><firstterm>stencil</firstterm></glossterm>
+   computes a container's value using neighboring data values.  Each
+   stencil consists of an indication of which neighboring values to
+   read and a function using those values.  For example, an averaging
+   stencil may access all neighbors, averaging them.  In &pooma;, we
+   represent a stencil using a function object having functions
+   indicating which neighboring values are used.  Stencil
+   computations are frequently used in solving partial differential
+   equations, image processing, and geometric modeling.</para>
+ 
+   <para>A <glossterm
+   linkend="glossary-relation"><firstterm>relation</firstterm></glossterm>
+   is a dependence among containers so the dependent container's
+   values are updated when its values are needed and any of its
+   related containers' values have changed.  A relation is specified
+   by a dependent container, independent containers, and a function
+   computing the dependent container's values using the independent
+   containers' values.  To avoid excess computation, the dependent
+   container's values are computed only when needed, e.g., for
+   printing or for computing the values of another dependent
+   container.  Thus, this computation is sometimes called <quote>lazy
+   evaluation</quote>.</para>
+  </section>
+ 
+ 
+  <section id="concepts-computation_environment">
+   <title>Computation Environment</title>
+ 
+   <para>A &pooma; program can execute on a wide variety of
+    computers.  The default <glossterm
+    linkend="glossary-sequential"><firstterm>sequential computing
+    environment</firstterm></glossterm> consists of one processor and
+    associated memory, as found on a personal computer.  In contrast,
+    a <glossterm
+    linkend="glossary-distributed"><firstterm>distributed computing
+    environment</firstterm></glossterm> may have multiple processors
+    and multiple distributed or shared memories.  For example, some
+    desktop computers have dual processors and shared memory, while a
+    large supercomputer may have thousands of processors, perhaps
+    with groups of eight sharing the same memory.</para>
+ 
+   <para>Using distributed computation requires three things:
+    <orderedlist>
+     <listitem>
+      <para>the programmer must declare how container domains will
+       be distributed,</para>
+     </listitem>
+     <listitem>
+      <para>&pooma; must be configured to use a communications
+       library, and </para>
+      </listitem>
+      <listitem>
+ 		  <para>the &pooma; executable must be run using the
+ 	library.</para>
+     </listitem>
+    </orderedlist>
+    All of these were illustrated in <xref
+    linkend="tutorial-array_distributed"></xref> and <xref
+    linkend="tutorial-field_distributed"></xref>.  <xref
+    linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
+    illustrates the &pooma; distributed computation model.  <xref
+    linkend="concepts-containers-distributed_declarations"></xref>
+    described how to declare containers with distributed domains.
+    Detailed instructions how to configure &pooma; for distributed
+    computation appear in <xref linkend="installation"></xref>.
+    Detailed instructions how to run distributed &pooma; executables
+    appear in <xref linkend="parallel"></xref>.  Here we present
+    three concepts for distributed computation: patches, context, and
+    a communication library.</para>
+ 
+   <para>A partition divides a container's domain into disjoint
+    <glossterm
+    linkend="glossary-patch"><firstterm>patch</firstterm></glossterm>es.
+    For distributed computation, the patches are distributed among
+    various processors, which compute the associated values.  As
+    illustrated in <xref
+    linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>,
+    each patch can be surrounded by guard layers.</para>
+ 
+   <para>A <glossterm
+    linkend="glossary-context"><firstterm>context</firstterm></glossterm>
+    is a collection of shared memory and processors that can execute
+    a program of a portion of a program.  It can have one or more
+    processors, but all these processors must access the same shared
+    memory.  Usually the computer and its operating system, not the
+    programmer, determine the available contexts.</para>
+ 
+   <para>A <glossterm
+    linkend="glossary-communication_library"><firstterm>communication
+    library</firstterm></glossterm> passes messages among contexts.
+    &pooma; uses the communication library to copy information among
+    contexts, all of which is hidden from both the programmer and the
+    user.  &pooma; works with the Message Passing Interface (&mpi;)
+    Communications Library (FIXME: xref linkend="mpi99", <ulink
+    url="http://www-unix.mcs.anl.gov/mpi/"></ulink>) and the &mm;
+    Shared Memory Library.</para>
+   </section>
+ </chapter>
Index: glossary.xml
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/glossary.xml,v
retrieving revision 1.1
diff -c -p -r1.1 glossary.xml
*** glossary.xml	2001/12/11 20:36:13	1.1
--- glossary.xml	2001/12/13 04:02:05
***************
*** 72,77 ****
--- 72,86 ----
     </glossdef>
    </glossentry>
  
+   <glossentry id="glossary-communication_library">
+    <glossterm>communication library</glossterm>
+    <glossdef>
+     <para>software library passing information among contexts, usually
+     using messages.</para>
+     <glossseealso otherterm="glossary-distributed">distributed computing environment</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
    <glossentry id="glossary-computing_environment">
     <glossterm>computing environment</glossterm>
     <glossdef>
***************
*** 137,142 ****
--- 146,165 ----
   <glossdiv id="glossary-d">
    <title>D</title>
  
+   <glossentry id="glossary-data_parallel">
+    <glossterm>data parallel</glossterm>
+    <glossdef>
+     <para>describes an expression representing a subset of a
+     container's values.  For example,
+     <statement>sin(&container;)</statement> is an expression
+     indicating that the <function>sin</function> is applied to each
+     value in container&nbsp;&container;.</para>
+     <glossseealso otherterm="glossary-element_wise">element wise</glossseealso>
+     <glossseealso otherterm="glossary-relation">relation</glossseealso>
+     <glossseealso otherterm="glossary-stencil">stencil</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
    <glossentry id="glossary-distributed">
     <glossterm>distributed computing environment</glossterm>
     <glossdef>
***************
*** 166,176 ****
     <glossterm>&dynamicarray;</glossterm>
     <glossdef>
      <para>a &pooma; container generalizing one-dimensional <link
! 								  linkend="glossary-array">&array;</link>s by supporting domain
      resizing at run-time.  It maps indices to values in constant-time
      access, ignoring the time to compute the values if applicable.
      &dynamicarray;s are <link
! 			       linkend="glossary-first_class">first-class object</link>s.</para>
      <glossseealso otherterm="glossary-array">&array;</glossseealso>
      <glossseealso otherterm="glossary-field">&field;</glossseealso>
     </glossdef>
--- 189,199 ----
     <glossterm>&dynamicarray;</glossterm>
     <glossdef>
      <para>a &pooma; container generalizing one-dimensional <link
!     linkend="glossary-array">&array;</link>s by supporting domain
      resizing at run-time.  It maps indices to values in constant-time
      access, ignoring the time to compute the values if applicable.
      &dynamicarray;s are <link
!     linkend="glossary-first_class">first-class object</link>s.</para>
      <glossseealso otherterm="glossary-array">&array;</glossseealso>
      <glossseealso otherterm="glossary-field">&field;</glossseealso>
     </glossdef>
***************
*** 180,195 ****
    <glossdiv id="glossary-e">
    <title>E</title>
  
    <glossentry id="glossary-engine">
     <glossterm>engine</glossterm>
     <glossdef>
      <para>stores and, if necessary, computes a <link
! 						      linkend="glossary-container">container</link>'s values.  These
!     can be specialized, e.g., to minimize storage when a domain has
!     few distinct values.  Separating a container and its storage also
!     permits <link linkend="glossary-view">views</link> of a container.</para>
!     <glossseealso otherterm="glossary-container">&engine;</glossseealso>
!     <glossseealso otherterm="glossary-view">view of a container</glossseealso>
     </glossdef>
    </glossentry>
  
--- 203,232 ----
    <glossdiv id="glossary-e">
    <title>E</title>
  
+   <glossentry id="glossary-element_wise">
+    <glossterm>element wise</glossterm>
+    <glossdef>
+     <para>describes accesses to individual values within a container.
+     For example, <statement>&container(i,j)</statement> represents one
+     particular value in the container&nbsp;&container;.</para>
+     <glossseealso otherterm="glossary-data_parallel">data parallel</glossseealso>
+     <glossseealso otherterm="glossary-relation">relation</glossseealso>
+     <glossseealso otherterm="glossary-stencil">stencil</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
    <glossentry id="glossary-engine">
     <glossterm>engine</glossterm>
     <glossdef>
      <para>stores and, if necessary, computes a <link
!     linkend="glossary-container">container</link>'s values.  These can
!     be specialized, e.g., to minimize storage when a domain has few
!     distinct values.  Separating a container and its storage also
!     permits <link linkend="glossary-view">views</link> of a
!     container.</para> <glossseealso
!     otherterm="glossary-container">&engine;</glossseealso>
!     <glossseealso otherterm="glossary-view">view of a
!     container</glossseealso>
     </glossdef>
    </glossentry>
  
***************
*** 221,227 ****
      also supports geometric computations such as the distance between
      two <link linkend="glossary-cell">cell</link>s and normals to a
      cell.  &field;s are <link
! 			       linkend="glossary-first_class">first-class object</link>s.</para>
      <glossseealso otherterm="glossary-array">&array;</glossseealso>
      <glossseealso otherterm="glossary-dynamicarray">&dynamicarray;</glossseealso>
     </glossdef>
--- 258,264 ----
      also supports geometric computations such as the distance between
      two <link linkend="glossary-cell">cell</link>s and normals to a
      cell.  &field;s are <link
!     linkend="glossary-first_class">first-class object</link>s.</para>
      <glossseealso otherterm="glossary-array">&array;</glossseealso>
      <glossseealso otherterm="glossary-dynamicarray">&dynamicarray;</glossseealso>
     </glossdef>
***************
*** 236,242 ****
      may be declared anywhere, stored in automatic variables, accessed
      anywhere, copied, and passed by both value and reference.
      &pooma; <link linkend="glossary-array">&array;</link> and <link
! 								     linkend="glossary-field">&field;</link> are first-class
      types.</para>
     </glossdef>
    </glossentry>
--- 273,279 ----
      may be declared anywhere, stored in automatic variables, accessed
      anywhere, copied, and passed by both value and reference.
      &pooma; <link linkend="glossary-array">&array;</link> and <link
!     linkend="glossary-field">&field;</link> are first-class
      types.</para>
     </glossdef>
    </glossentry>
***************
*** 373,380 ****
    <glossentry id="glossary-patch">
     <glossterm>patch</glossterm>
     <glossdef>
!     <para>
! ERE</para>
      <glossseealso otherterm="glossary-partition">partition</glossseealso>
      <glossseealso otherterm="glossary-guard_layer">guard layer</glossseealso>
      <glossseealso otherterm="glossary-domain">domain</glossseealso>
--- 410,418 ----
    <glossentry id="glossary-patch">
     <glossterm>patch</glossterm>
     <glossdef>
!     <para>subset of a container's domain with values computed by a
!     particular context.  A partition splits a domain into patches.  It
!     may be surrounded by external and internal guard layers.</para>
      <glossseealso otherterm="glossary-partition">partition</glossseealso>
      <glossseealso otherterm="glossary-guard_layer">guard layer</glossseealso>
      <glossseealso otherterm="glossary-domain">domain</glossseealso>
*************** ERE</para>
*** 417,422 ****
--- 455,475 ----
      <glossseealso otherterm="glossary-interval">interval</glossseealso>
     </glossdef>
    </glossentry>
+ 
+   <glossentry id="glossary-relation">
+    <glossterm>relation</glossterm>
+    <glossdef>
+     <para>dependence between a dependent container and one or more
+     independent containers and an associated function.  If a dependent
+     container's values are needed and one or more of the independent
+     containers' values have changed, the dependent container's values
+     are computed using the function and the independent containers'
+     values.  Relations implement <quote>lazy evaluation</quote>.</para>
+     <glossseealso otherterm="glossary-data_parallel">data parallel</glossseealso>
+     <glossseealso otherterm="glossary-element_wise">element wise</glossseealso>
+     <glossseealso otherterm="glossary-stencil">stencil</glossseealso>
+    </glossdef>
+   </glossentry>
   </glossdiv>
  
   <glossdiv id="glossary-s">
*************** ERE</para>
*** 427,435 ****
     <glossdef>
      <para>a computing environment with one processor and associated
      memory.  Only one processor executes a program even if the
!     conmputer itself has multiple processors.</para>
      <glossseealso otherterm="glossary-computing_environment">computing environment</glossseealso>
      <glossseealso otherterm="glossary-distributed">distributed computing environment</glossseealso>
     </glossdef>
    </glossentry>
  
--- 480,503 ----
     <glossdef>
      <para>a computing environment with one processor and associated
      memory.  Only one processor executes a program even if the
!     computer itself has multiple processors.</para>
      <glossseealso otherterm="glossary-computing_environment">computing environment</glossseealso>
      <glossseealso otherterm="glossary-distributed">distributed computing environment</glossseealso>
+    </glossdef>
+   </glossentry>
+ 
+   <glossentry id="glossary-stencil">
+    <glossterm>stencil</glossterm>
+    <glossdef>
+     <para>set of values neighboring a container value and a function
+     using those values to compute it.  For example, the stencil in a
+     two-dimensional Conway game of life consists of a value's eight
+     neighbors and a function that sets the value to
+     <quote>live</quote> if it is already live or it has exactly three
+     live neighbors.</para>
+     <glossseealso otherterm="glossary-data_parallel">data parallel</glossseealso>
+     <glossseealso otherterm="glossary-element_wise">element wise</glossseealso>
+     <glossseealso otherterm="glossary-relation">relation</glossseealso>
     </glossdef>
    </glossentry>
  
Index: manual.xml
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/manual.xml,v
retrieving revision 1.1
diff -c -p -r1.1 manual.xml
*** manual.xml	2001/12/11 20:36:13	1.1
--- manual.xml	2001/12/13 04:02:07
***************
*** 154,159 ****
--- 154,161 ----
    <!-- Field data-parallel Doof2d implementation -->
  <!ENTITY doof2d-field-distributed SYSTEM "./programs/Doof2d-Field-distributed-annotated.cpp">
    <!-- Field data-parallel distributed Doof2d implementation -->
+ <!ENTITY concepts-chapter SYSTEM "concepts.xml">
+   <!-- Pooma concepts chapter -->
  <!ENTITY glossary-chapter SYSTEM "glossary.xml">
    <!-- glossary -->
  <!ENTITY tutorial-chapter SYSTEM "tutorial.xml">
***************
*** 345,1008 ****
  
    </chapter>
  
-   &tutorial-chapter;
- 
- 
-   <chapter id="concepts">
-    <title>Overview of &pooma; Concepts</title>
- 
-    <para>FIXME: How does multi-threaded computation fit into the
-    model?</para>
- 
-    <para>In the previous chapter, we presented several different
-    implementations of the &doof2d; simulation program.  The
-    implementations illustrate the various containers, computation
-    syntaxes, and computation environments that &pooma; supports.  In
-    this chapter, we describe the concepts associated with each of
-    these three categories.  Specific details needed by programmers are
-    deferred to later chapters.</para>
  
!    <table frame="none" colsep="0" rowsep="0" tocentry="1"
!      orient="port" pgwide="0" id="concepts-table">
!      <title>&pooma; Implementation Concepts</title>
!      <tgroup cols="3" align="center">
!       <thead>
!        <row>
!         <entry>Container</entry>
! 	<entry>Computation Syntax</entry>
! 	<entry>Computation Environment</entry>
!        </row>
!       </thead>
!       <tbody>
!        <row>
! 	<entry>&array;</entry>
! 	<entry>element-wise</entry>
! 	<entry>sequential</entry>
!        </row>
!        <row>
! 	<entry>&dynamicarray;</entry>
! 	<entry>data-parallel</entry>
! 	<entry>distributed</entry>
!        </row>
!        <row>
! 	<entry>&field;</entry>
! 	<entry>stencil-based</entry>
! 	<entry></entry>
!        </row>
!        <row>
! 	<entry>&tensor;</entry>
! 	<entry>relational</entry>
! 	<entry></entry>
!        </row>
!        <row>
! 	<entry>&matrix;</entry>
! 	<entry></entry>
! 	<entry></entry>
!        </row>
!        <row>
! 	<entry>&vector;</entry>
! 	<entry></entry>
! 	<entry></entry>
!        </row>
!       </tbody>
!      </tgroup>
!     </table>
! 
!    <para>The most important &pooma; concepts can be grouped into three
!    separate categories:
!    <variablelist>
!     <varlistentry>
!       <term>container</term>
!       <listitem>
!        <para>data structure holding one or more values and addressed
!        by indices</para>
!       </listitem>
!      </varlistentry>
!      <varlistentry>
!       <term>computation syntax</term>
!       <listitem>
!        <para>styles of expressing computations</para>
!       </listitem>
!      </varlistentry>
!      <varlistentry>
!       <term>computation environment</term>
!       <listitem>
!        <para>description of resources for computing, e.g., single
!        processor or multi-processor</para>
!       </listitem>
!      </varlistentry>
!     </variablelist>
!     See <xref linkend="concepts-table"></xref>.  Many &pooma; programs
!     select one possibility from each column.  For example, <xref
!     linkend="tutorial-array_stencil-doof2d"></xref> used a &array;
!     container and stencils for sequential computation, while <xref
!     linkend="tutorial-field_distributed-doof2d"></xref> used a &field;
!     container and data-parallel statements with distributed
!     computation.  A program may use multiple containers and various
!     computation syntax, but the computation environment either has
!     distributed processors or not.</para>
! 
!    <para>In the rest of this chapter, we explore these three
!    categories.  First, we describe &pooma; containers, illustrating
!    the purposes of each, and explaining the concepts needed to declare
!    them.  Then, we describe the different computation syntaxes and
!    finally distributed computation concepts.</para>
! 
! 
!    <section id="concepts-containers">
!     <title>&pooma; Containers</title>
! 
!     <para>Most &pooma; programs use <firstterm>containers</firstterm>
!     to store groups of values.  &pooma; containers are objects that
!     store other objects.  They control allocation and deallocation of
!     and access to these objects.  They are a generalization of &c;
!     arrays, but &pooma; containers are first-class objects so they can
!     be used directly in expressions.  They are similar to &cc;
!     containers such as <type>vector</type>, <type>list</type>, and
!     <type>stack</type>.  See <xref
!     linkend="concepts-containers-table"></xref> for a summary of the
!     containers.</para>
! 
!     <para>This chapter describes many concepts, not all of which are
!     needed to begin programming with the &pooma; Toolkit.  Below we
!     introduce the different categories of concepts.  After that, we
!     introduce the different &pooma;'s containers and describe how to
!     choose the appropriate one for a particular task.  <xref
!     linkend="concepts-sequential_containers-declarations-dependences"></xref>
!     indicates which concepts must be understood when declaring a
!     particular container.  All of these concepts are described in
!     <xref
!     linkend="concepts-containers-sequential_declarations"></xref> and
!     <xref
!     linkend="concepts-containers-distributed_declarations"></xref>.
!     Use this figure to decide which concepts in the former are
!     relevant.  Reading the latter section is necessary only if
!     computing using multiple processors.  The programs in the previous
!     chapter illustrate many of these concepts.</para>
! 
!     <table frame="none" colsep="0" rowsep="0" tocentry="1"
! 	   orient="port" pgwide="0" id="concepts-containers-table">
!      <title>&pooma; Container Summary</title>
!      <tgroup cols="2" align="left">
!       <tbody>
!        <row>
! 	<entry><glossterm
! 	linkend="glossary-array">&array;</glossterm></entry>
! 	<entry>container mapping indices to values and that may be
! 	used in expressions</entry>
!        </row>
!        <row>
! 	<entry><glossterm
! 	linkend="glossary-dynamicarray">&dynamicarray;</glossterm></entry>
! 	<entry>one-dimensional &array; whose <glossterm
! 	linkend="glossary-domain">domain</glossterm> can be dynamically
! 	resized</entry>
!        </row>
!        <row>
! 	<entry><glossterm linkend="glossary-field">&field;</glossterm></entry>
! 	<entry>container mapping indices to one or more values and
! 	residing in multi-dimensional space</entry>
!        </row>
!        <row>
! 	<entry><glossterm
! 	linkend="glossary-tensor">&tensor;</glossterm></entry>
! 	<entry>multi-dimensional mathematical tensor</entry>
!        </row>
!        <row>
! 	<entry><glossterm
! 	linkend="glossary-matrix">&matrix;</glossterm></entry>
! 	<entry>two-dimensional mathematical matrix</entry>
!        </row>
!        <row>
! 	<entry><glossterm
! 	linkend="glossary-vector">&vector;</glossterm></entry>
! 	<entry>multi-dimensional mathematical vector</entry>
!        </row>
!       </tbody>
!      </tgroup>
!     </table>
! 
! <!-- FIXME: Want firstterm around array. -->
! 
!     <para>A &pooma; <glossterm
!     linkend="glossary-array">array;</glossterm>, generalizing a &c;
!     array, maps indices to values.  Given a index or position in an
!     &array;'s domain, it returns the associated value, either by
!     returning a stored value or by computing it.  The use of indices,
!     which are usually ordered tuples, permits constant-time access
!     although computing a particular value may require significant
!     time.  In addition to the functionality provided by &c; arrays,
!     the &array; class automatically handles memory allocation and
!     deallocation, supports a wider variety of assignments, and can be
!     used in expressions.  For example, the addition of two arrays can
!     be assigned to an array and the product of a scalar element and an
!     array is permissible.</para>
! 
! <!-- FIXME: Want firstterm around dynamicarray. -->
! 
!     <para>A &pooma; <glossterm
!     linkend="glossary-dynamicarray">&dynamicarray;</glossterm> extends
!     &array; capabilities to support a dynamically-changing domain but
!     is restricted to only one dimension.  When the &dynamicarray; is
!     resized, its values are preserved.</para>
! 
! <!-- FIXME: Want firstterm around field. -->
! 
!     <para>A &pooma; <glossterm>&field;</glossterm> is an &array; with
!     spatial extent.  Each domain consists of <glossterm
!     linkend="glossary-cell"><firstterm>cell</firstterm></glossterm>s
!     in one-, two-, or three-dimensional space.  Although indexed
!     similarly to &array;s, each cell may contain multiple values and
!     multiple materials.  A &field;'s <glossterm
!     linkend="glossary-mesh">mesh</glossterm> stores its spatial
!     characteristics and can map yield, e.g., a point contained in a
!     cell, the distance between two cells, and a cell's normals.  A
!     &field; should be used whenever geometric or spatial computations
!     are needed, multiple values per index are desired, or a
!     computation involves more than one material.</para>
! 
! <!-- FIXME: Want firstterm around tensor. -->
! 
!     <para>A <glossterm linkend="glossary-tensor">&tensor;</glossterm>
!     implements a multi-dimensional mathematical tensor.  Since it is a
!     first-class type, it can be used in expressions such as
!     adding two &tensor;s.</para>
! 
! <!-- FIXME: Want firstterm around matrix. -->
! 
!     <para>A <glossterm linkend="glossary-matrix">&matrix;</glossterm>
!     implements a two-dimensional mathematical matrix.  Since it is a
!     first-class type, it can be used in expressions such as
!     multiplying matrices and assignments to matrices.</para>
! 
! <!-- FIXME: Want firstterm around vector. -->
! 
!     <para>A <glossterm linkend="glossary-vector">&vector;</glossterm>
!     implements a multi-dimensional mathematical vector, which is an
!     ordered tuple of components.  Since it is a first-class type, it
!     can be used in expressions such as adding two &vector;s and
!     multiplying a &matrix; and a &vector;.</para>
! 
!     <para>The data of an &array;, &dynamicarray;, or &field; can be
!     viewed using more than one container by taking a view.  A
!     <glossterm
!     linkend="glossary-view"><firstterm>view</firstterm></glossterm> of
!     an existing container&nbsp;&container; is a container whose domain
!     is a subset of&nbsp;&container;.  The subset can equal the
!     original domain.  A view acts like a reference in that changing
!     any of the view's values also changes the original container's and
!     vice versa.  While users sometimes explicitly create views, they
!     are perhaps more frequently created as temporaries in expressions.
!     For example, if <varname>A</varname> is an &array; and
!     <varname>I</varname> is a domain, <statement>A(I) -
!     A(I-1)</statement> forms the difference between adjacent
!     values.</para>
! 
! 
!     <section id="concepts-containers-choosing">
!      <title>Choosing a Container</title>
! 
!      <para>The two most commonly used &pooma; containers are &array;s
!      and &field;s.  <xref
!      linkend="concepts-containers-choice_table"></xref> contains a
!      decision tree describing how to choose an appropriate
!      container.</para>
! 
!     <table frame="none" colsep="0" rowsep="0" tocentry="1"
! 	   orient="port" pgwide="0" id="concepts-containers-choice_table">
!      <title>Choosing a &pooma; Container</title>
!      <tgroup cols="2" align="left">
!       <tbody>
!        <row>
! 	<entry>If modeling mathematical entries,</entry>
! 	<entry>use a &vector;, &matrix;, or &tensor;.</entry>
!        </row>
!        <row>
! 	<entry>If indices and values reside in multi-dimensional space
! 	&space;,</entry>
! 	<entry>use a &field;.</entry>
!        </row>
!        <row>
! 	<entry>If there are multiple values per index,</entry>
! 	<entry>use a &field;.</entry>
!        </row>
!        <row>
! 	<entry>If there are multiple materials participating in the same computation,</entry>
! 	<entry>use a &field;.</entry>
!        </row>
!        <row>
! 	<entry>If the domain's size dynamically changes and is one-dimensional,</entry>
! 	<entry>use a &dynamicarray;.</entry>
!        </row>
!        <row>
! 	<entry>Otherwise</entry>
! 	<entry>use an &array;.</entry>
!        </row>
!       </tbody>
!      </tgroup>
!     </table>
! 
!     </section><!-- choosing among containers -->
  
  
-     <section id="concepts-containers-sequential_declarations">
-      <title>Declaring Sequential Containers</title>
  
!      <figure float="1" id="concepts-sequential_containers-declarations-dependences">
!       <title>Concepts For Declaring Containers</title>
!       <mediaobject>
!        <imageobject>
! 	<imagedata fileref="figures/concepts.111" format="EPS" align="center"></imagedata>
!        </imageobject>
!        <textobject>
! 	<phrase>concepts involved in declaring containers</phrase>
!        </textobject>
!       </mediaobject>
!      </figure>
! 
!      <para>In the previous sections, we introduced the &pooma;
!      containers and described how to choose one appropriate for a
!      given task.  In this section, we describe the concepts involved
!      in declaring them.  Concepts specific to distributed computation
!      are described in the next section.</para>
! 
!      <para><xref
!      linkend="concepts-sequential_containers-declarations-dependences"></xref>
!      illustrates the containers and the concepts involved in their
!      declarations.  The containers are listed in the top row.  Lines
!      connect these containers to the components necessary for their
!      declarations.  For example, an &array; declaration requires an
!      engine and a layout.  These, in turn, depend on other &pooma;
!      concepts.  Declarations necessary only for distributed, or
!      multiprocessor, computation are surrounded by dashed lines.  You
!      can use these dependences to indicate the concepts needed for a
!      particular container.</para>
! 
!      <para>An <glossterm
!      linkend="glossary-engine"><firstterm>engine</firstterm></glossterm>
!      stores and, if necessary, computes a container's values.  A
!      container has one or more engines.  The separation of a container
!      and its storage permits optimizing a program's space
!      requirements.  For example, a container returning the same value
!      for all indices can use a constant engine, which need only store
!      one value for the entire domain.  A &compressiblebrick; engine
!      reduces its space requirements to a constant whenever all its
!      values are the same.  The separation also permits taking <link
!      linkend="glossary-view">view</link>s of containers without
!      copying storage.</para>
! 
!      <figure float="1" id="concepts-containers-declarations-computational_implementation">
!       <title>&array; and &field; Mathematical and Computational Concepts</title>
!       <mediaobject>
!        <imageobject>
! 	<imagedata fileref="figures/concepts.101" format="EPS" align="center"></imagedata>
!        </imageobject>
!        <textobject>
! 	<phrase>maps from indices to values</phrase>
!        </textobject>
!       </mediaobject>
!      </figure>
! 
!      <para>A <glossterm
!      linkend="glossary-layout"><firstterm>layout</firstterm></glossterm>
!      maps <link linkend="glossary-domain">domain</link> indices to the
!      processors and computer memory used by a container's engines.
!      See <xref
!      linkend="concepts-containers-declarations-computational_implementation"></xref>.
!      A computer computes a container's values using a processor and
!      memory.  The layout specifies the processor(s) and memory to use
!      for each particular index.  A container's layout for a
!      uniprocessor implementation consists of its domain, the
!      processor, and its memory.  For a multi-processor implementation,
!      the layout maps portions of the domain to (possibly different)
!      processors and memory.</para>
! 
!      <para>A &field;'s <glossterm
!      linkend="glossary-mesh"><firstterm>mesh</firstterm></glossterm>
!      maps <link linkend="glossary-domain">domain</link> indices to
!      geometric values in &space; such as distance between cells, edge
!      lengths, and normals to cells.  In other words, it provides a
!      &field;'s spatial extent.  See also <xref
!      linkend="concepts-containers-declarations-computational_implementation"></xref>.
!      Different mesh types may support different geometric
!      values.</para>
! 
!      <para>A mesh's <glossterm
!      linkend="glossary-corner_position"><firstterm>corner
!      position</firstterm></glossterm> specifies the <link
!      linkend="glossary-point">point</link> in &space; corresponding to
!      the lower, left corner of its <link
!      linkend="glossary-domain">domain</link>.  Combining this, the
!      domain, and the cell size fully specifies the mesh's map from
!      indices to &space;.</para>
! 
!      <para>A mesh's <glossterm
!      linkend="glossary-cell_size"><firstterm>cell
!      size</firstterm></glossterm> specifies the spatial dimensions of
!      a &field; <link linkend="glossary-cell">cell</link>, e.g., its
!      width, height, and depth, in&nbsp;&space;.  Combining this, the
!      domain, and the corner position fully specifies the mesh's map
!      from indices to &space;.</para>
! 
!      <para>A <glossterm
!      linkend="glossary-domain"><firstterm>domain</firstterm></glossterm>
!      is a set of points on which a container can define values.  An
!      <glossterm
!      linkend="glossary-interval"><firstterm>interval</firstterm></glossterm>
!      consists of all integral points between two values.  It is
!      frequently represented using mathematical interval notation [a,b]
!      even though it contains only the integral points, e.g., a, a+1,
!      a+2, &hellip;, b.  The concept is generalized to multiple
!      dimensions by forming tensor product of intervals, i.e., all the
!      integral tuples in an &n;-dimensional space.  For example, the
!      two-dimensional containers in the previous chapter are defined on
!      a two-dimensional domain with the both dimensions' spanning the
!      interval [0,n).  A <glossterm
!      linkend="glossary-stride"><firstterm>stride</firstterm></glossterm>
!      is a subset of an interval consisting of regularly-spaced
!      points.  A <glossterm
!      linkend="glossary-range"><firstterm>range</firstterm></glossterm>
!      is a subset of an interval formed by the tensor product of strides.
!      A <glossterm
!      linkend="glossary-region"><firstterm>region</firstterm></glossterm>
!      represents a continuous &n;-dimensional domain.</para>
!     </section>
! 
! 
!     <section id="concepts-containers-distributed_declarations">
!      <title>Declaring Distributed Containers</title>
! 
!      <para>In the previous section, we introduced the concepts
!      important when declaring containers for use on uniprocessor
!      computers.  When using multi-processor computers, we augment
!      these concepts with those for distributed computation.  Reading
!      this section is important only for running the same program on
!      multiple processors.  Many of these concepts were introduced in
!      <xref linkend="tutorial-array_distributed"></xref> and <xref
!      linkend="tutorial-field_distributed"></xref>.  <xref
!      linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
!      illustrates the &pooma; distributed computation model.  In this
!      section, we concentrate on the concepts necessary to declare a
!      distributed container.</para>
! 
!      <para>As we noted in <xref
!      linkend="tutorial-array_distributed"></xref>, a &pooma;
!      programmer must specify how each container's domain should be
!      distributed among the available processors and memory spaces.
!      Using this information, the Toolkit automatically distributes the
!      data among the available processors and handles any required
!      communication among them.  The three concepts necessary for
!      declaring distributed containers are a partition, a guard layer,
!      and a context mapper tag.</para>
! 
!      <para>A <glossterm
!      linkend="glossary-partition"><firstterm>partition</firstterm></glossterm>
!      specified how to divide a container's domain into distributed
!      pieces.  For example, the partition illustrated in <xref
!      linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
!      would divide a two-dimensional domain into three equally-sized
!      pieces along the x-dimension and two equally-sized pieces along
!      the y-dimension.  Partitions can be independent of the size of
!      container's domain.  The example partition will work on any
!      domain as long as the size of its x-dimension is a multiple of
!      three.  A domain is separated into disjoint <link
!      linkend="glossary-patch">patch</link>es.</para>
! 
!      <para>A <glossterm
!      linkend="glossary-guard_layer"><firstterm>guard
!      layer</firstterm></glossterm> is <quote>extra</quote> domain
!      surrounding each patch.  This region has read-only values.  An
!      <glossterm
!      linkend="glossary-external_guard_layer"><firstterm>external guard
!      layer</firstterm></glossterm> specifies values surrounding the
!      domain.  Its presence eases computation along the domain's edges
!      by permitting the same computations as for more internal
!      computations.  An <glossterm
!      linkend="glossary-internal_guard_layer"><firstterm>internal guard
!      layer</firstterm></glossterm> duplicates values from adjacent
!      patches so communication with adjacent patches need not occur
!      during a patch's computation.  The use of guard layers is an
!      optimization; using external guard layers eases programming and
!      using internal guard layers reduces communication among
!      processors.  Their use is not required.</para>
! 
!      <para>A <glossterm
!      linkend="glossary-context_mapper"><firstterm>context
!      mapper</firstterm></glossterm> indicates how a container's
!      patches are mapped to processors and shared memory.  For example,
!      the &distributedtag; indicates that the patches should be
!      distributed among the processors so each patch occurs once in the
!      entire computation.  The &replicatedtag; indicates that the
!      patches should be replicated among the processors so each
!      processing unit has its own copy of all the patches.  While it
!      could be wasteful to have different processors perform the same
!      computation, replicating a container can reduce possibly more
!      expensive communication costs.</para>
!     </section>
! 
! 
!     <section>
!      <title>????Computation Syntax????</title>
! 
!      <para>UNFINISHED</para>
!     </section>
! 
! 
!     <section id="concepts-computation_environment">
!      <title>Computation Environment</title>
! 
!      <para>A &pooma; program can execute on a wide variety of
!      computers.  The default <glossterm
!      linkend="glossary-sequential"><firstterm>sequential computing
!      environment</firstterm></glossterm> consists of one processor and
!      associated memory, as found on a personal computer.  In contrast,
!      a <glossterm
!      linkend="glossary-distributed"><firstterm>distributed computing
!      environment</firstterm></glossterm> may have multiple processors
!      and multiple distributed or shared memories.  For example, some
!      desktop computers have dual processors and shared memory.  A
!      large supercomputer may have thousands of processors, perhaps
!      with groups of eight sharing the same memory.</para>
! 
!      <para>Using distributed computation requires three things: the
!      programmer must declare how container domains will be
!      distributed, &pooma; must be configured to use a communications
!      library, and the &pooma; executable must be run using the
!      library.  All of these were illustrated in <xref
!      linkend="tutorial-array_distributed"></xref> and <xref
!      linkend="tutorial-field_distributed"></xref>.  <xref
!      linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
!      illustrates the &pooma; distributed computation model.  <xref
!      linkend="concepts-containers-distributed_declarations"></xref>
!      described how to declare containers with distributed domains.
!      Detailed instructions how to configure &pooma; for distributed
!      computation appear in <xref linkend="installation"></xref>.  More
!      detailed instructions how to run distributed &pooma; executables
!      appear in <xref linkend="parallel"></xref>.  Here we present
!      three concepts for distributed computation: context, layout, and
!      a communication library.
! 
!      A <glossterm
!      linkend="glossary-context"><firstterm>context</firstterm></glossterm>
!      is a collection of shared memory and processors that can execute
!      a program of a portion of a program.  It can have one or more
!      processors, but all these processors must access the same shared
!      memory.  Usually the computer and its operating system, not the
!      programmer, determine the available contexts.</para>
  
!      <para>
! HERE</para>
  
  <!-- HERE -->
-     </section>
- 
-    </section>
- 
- <section>
- <title>Extraneous Material</title>
- 
-    <para>Describe the software application layers similar to
-    <filename>papers/SCPaper-95.html</filename> and "Short Tour of
-    &pooma;" in <filename>papers/SiamOO98_paper.ps</filename>.
-    Section&nbsp;2.2, "Why a Framework?," of
-    <filename>papers/pooma.ps</filename> argues why a layered approach
-    eases use.  Section&nbsp;3.1, "Framework Layer Description,"
-    describes the five layers.</para>
  
!    <para>FINISH: Write short glossary entries for each of these.</para>
! 
!    <para>FINISH: Look through the source code to ensure all main
!    concepts are listed.</para>
! 
!    <para>Here are (preliminary) &pooma; equations:</para>
  
!    <table frame="none" colsep="0" rowsep="0" tocentry="1"
! 	  orient="port" pgwide="0">
!     <title>&pooma; <quote>Equations</quote></title>
  
!     <tgroup cols="1" align="center">
!      <tbody>
!       <row>
!        <entry><para>field = data + materials + centering + layout + mesh</para></entry>
!        </row>
!       <row>
!        <entry><para>map from space to values</para></entry>
!       </row>
!       <row>
!        <entry><para>array = data + layout</para></entry>
!        </row>
!       <row>
!        <entry><para>map from indices to values</para></entry>
!       </row>
!       <row>
!        <entry><para>mesh = layout + origin + spacings</para></entry>
!        </row>
!       <row>
!        <entry><para>distribute domain through physical space</para></entry>
!       </row>
!       <row>
!        <entry><para>layout = domain + partition + context_mapper_tag (distributed/replicated)</para></entry>
!        </row>
!       <row>
!        <entry><para>distribute domain's blocks among processors/contexts</para></entry>
!       </row>
!       <row>
!        <entry><para>partition = blocks + guard layers</para></entry>
!        </row>
!       <row>
!        <entry><para>split domain into blocks</para></entry>
!       </row>
!       <row>
!        <entry><para>domain = newDomain</para></entry>
!        </row>
!       <row>
!        <entry><para>space of permissible indices</para></entry>
!       </row>
!      </tbody>
!     </tgroup>
!    </table>
  
  
-    <para>FINISH: Following is a first try at describing the &pooma;
-    abstraction layers.  See also paper illustration.</para>
  
!    <table frame="none" colsep="0" rowsep="0" tocentry="1"
! 	  orient="port" pgwide="0">
!     <title>&pooma; Abstraction Layers</title>
  
!     <tgroup cols="1" align="center">
!      <tbody>
!       <row>
!        <entry><para>application program</para></entry>
!       </row>
!       <row>
!        <entry><para>&array; &field; (should have
!        <type>FieldEngine</type> under it)</para></entry>
!       </row>
!       <row>
!        <entry><para>&engine;</para></entry>
!       </row>
!       <row>
!        <entry><para>evaluators</para></entry>
!       </row>
!      </tbody>
!     </tgroup>
!    </table>
  
-    <para>FINISH: How does parallel execution fit in?</para>
  
!    <para>FINISH: Should we also name and describe each layer?</para>
!    </section>
  
  
!    <section id="concepts-data_parallel">
!     <title>Data-Parallel Statements</title>
  
!     <para>Can we use "An Overview of &pete;" from
      <filename>papers/PETE_DDJ/ddj_article.html</filename> or is this
      too low-level?</para>
  
!     <para>Section&nbsp;3.2.1 of <filename>papers/pooma.ps</filename>
      gives a simple example of data-parallel expression.  It also has a
      paragraph introducing data-parallel operations and selecting
      subsets of domains.  Section&nbsp;3.4 describes the Chained
--- 347,424 ----
  
    </chapter>
  
  
!   &tutorial-chapter;
  
+   &concepts-chapter;
  
  
!   <chapter id="sequential">
!    <title>Writing Sequential Programs</title>
  
!    <para>QUESTIONS: How do I arrange this section?  What material do I
!    include?  What other books or models can I follow?</para>
  
  <!-- HERE -->
  
!    <para>&pooma; can reorder computations to permit more efficient
!    computation.  When running a sequential program, reordering may
!    permit omission of unneeded computations.  For example, if only
!    values from a particular field are printed, only computations
!    involving the field and containers dependent on it need to occur.
!    When running a distributed program, reordering may permit
!    computation and communication among processors to overlap.  &pooma;
!    automatically tracks dependences between data-parallel expressions,
!    ensuring correct ordering.  It does not track statements accessing
!    particular &array; and &field; values so the programmer must
!    precede these statements with calls to
!    <function>Pooma::blockAndEvaluate()</function>.  Each call forces
!    the executable to wait until all computation has completed.  Thus,
!    the desired values are known to be available.  In practice, some
!    calls to <function>Pooma::blockAndEvaluate</function> may not be
!    necessary, but omitting them requires knowledge of &pooma;'s
!    dependence computations, so the &author; recommends calling
!    <function>Pooma::blockAndEvaluate</function> before each access to
!    a particular value in an &array; or &field;.  Omitting a necessary
!    call may lead to a race condition.  See <xref
!    linkend="debugging_profiling-missing_blockandevaluate"></xref> for
!    instructions how to diagnose and eliminate these race conditions.</para>
  
!    <para>UNFINISHED</para>
  
!    <section id="sequential-benchmarks">
!     <title>&benchmark; Programs</title>
  
+     <para>Define a &benchmark; program vs. an example or an
+     executable.  Provide a short overview of how to run these
+     programs.  Provide an overview of how to write these programs.
+     See <filename
+     class="headerfile">src/Utilities/Benchmark.h</filename>.</para>
+    </section>
  
  
!    <section id="sequential-inform">
!     <title>Using <type>Inform</type>s for Output</title>
  
!     <para>UNFINISHED</para>
!    </section>
  
  
!    <section>
!     <title>Miscellaneous</title>
  
+     <para>Section&nbsp;3, "Domains and Views," of
+     <filename>papers/iscope98.pdf</filename> describes five types of
+     domains.</para>
  
!     <section id="concepts-data_parallel">
!      <title>Data-Parallel Statements</title>
  
!      <para>Can we use "An Overview of &pete;" from
      <filename>papers/PETE_DDJ/ddj_article.html</filename> or is this
      too low-level?</para>
  
!      <para>Section&nbsp;3.2.1 of <filename>papers/pooma.ps</filename>
      gives a simple example of data-parallel expression.  It also has a
      paragraph introducing data-parallel operations and selecting
      subsets of domains.  Section&nbsp;3.4 describes the Chained
*************** HERE</para>
*** 1013,1019 ****
      <para>From Section&nbsp;4 of
      <filename>papers/SiamOO98_paper.ps</filename>:</para>
  
!     <para>This version of &pete; reduces compile time of user codes
      and utilizes compile-time knowledge of expression &domain;s for
      better optimization.  For example, more efficient loops for
      evaluating an expression can be generated if &pete; knows that the
--- 429,435 ----
      <para>From Section&nbsp;4 of
      <filename>papers/SiamOO98_paper.ps</filename>:</para>
  
!      <para>This version of &pete; reduces compile time of user codes
      and utilizes compile-time knowledge of expression &domain;s for
      better optimization.  For example, more efficient loops for
      evaluating an expression can be generated if &pete; knows that the
*************** HERE</para>
*** 1028,1044 ****
      &pete; material.</para>
     </section>
  
     <section>
!     <title>Containers</title>
  
!     <section id="concepts-containers-array">
       <title>&array;</title>
  
       <blockquote>
!       <attribution>Section&nbsp;4 "Future Improvements in
       &pooma;&nbsp;II" of
       <filename>papers/SiamOO98_paper.ps</filename></attribution>
!       <para>
       An &array; can be thought of as a map from one &domain; to
       another.&hellip; &array;s depend only on the interface of
       &domain;s.  Thus, a subset of view of an &array; can be
--- 444,461 ----
      &pete; material.</para>
     </section>
  
+ 
     <section>
! 	     <title>Containers</title>
  
! 	     <section id="concepts-containers-array">
       <title>&array;</title>
  
       <blockquote>
! 	     <attribution>Section&nbsp;4 "Future Improvements in
       &pooma;&nbsp;II" of
       <filename>papers/SiamOO98_paper.ps</filename></attribution>
! 	     <para>
       An &array; can be thought of as a map from one &domain; to
       another.&hellip; &array;s depend only on the interface of
       &domain;s.  Thus, a subset of view of an &array; can be
*************** HERE</para>
*** 1058,1064 ****
        code. An Array maps a fairly arbitrary input domain to an
        arbitrary range of outputs. When used by itself, an &array;
        object <varname>A</varname> refers to all of the values in its
!       domain. Element-wise mathematical operations or functions can be
        applied to an array using straightforward notation, like A + B
        or sin(A). Expressions involving Array objects are themselves
        Arrays. The operation A(d), where d is a domain object that
--- 475,481 ----
        code. An Array maps a fairly arbitrary input domain to an
        arbitrary range of outputs. When used by itself, an &array;
        object <varname>A</varname> refers to all of the values in its
! 				  domain. Element-wise mathematical operations or functions can be
        applied to an array using straightforward notation, like A + B
        or sin(A). Expressions involving Array objects are themselves
        Arrays. The operation A(d), where d is a domain object that
*************** HERE</para>
*** 1084,1090 ****
        indexed.</para>
  
       <para>Fortran arrays are dense and the elements are arranged
!       according to column-major conventions. Therefore, X(i1,i2)
        refers to element number i1-1+(i2-1)*numberRowsInA. However, as
        Fig. 1 shows, Fortran-style "Brick" storage is not the only
        storage format of interest to scientific programmers. For
--- 501,507 ----
        indexed.</para>
  
       <para>Fortran arrays are dense and the elements are arranged
! 		   according to column-major conventions. Therefore, X(i1,i2)
        refers to element number i1-1+(i2-1)*numberRowsInA. However, as
        Fig. 1 shows, Fortran-style "Brick" storage is not the only
        storage format of interest to scientific programmers. For
*************** HERE</para>
*** 1103,1116 ****
  
       <para>The &pooma; &array; Class Template</para>
  
!     <para>Next we describe &pooma;'s model of the Array concept, the
       Array class template. The three most important requirements from
       the point of view of overall design are: (1) arbitrary domain,
       (2) arbitrary range, and (3) polymorphic indexing. These express
       themselves in the template parameters for the &pooma; Array
       class. The template
       <programlisting>
!      template &lt;int Dim, class T = double, class EngineTag = Brick&gt;
       class Array;
       </programlisting>
       is a specification for creating a set of classes all named
--- 520,533 ----
  
       <para>The &pooma; &array; Class Template</para>
  
!      <para>Next we describe &pooma;'s model of the Array concept, the
       Array class template. The three most important requirements from
       the point of view of overall design are: (1) arbitrary domain,
       (2) arbitrary range, and (3) polymorphic indexing. These express
       themselves in the template parameters for the &pooma; Array
       class. The template
       <programlisting>
! 		      template &lt;int Dim, class T = double, class EngineTag = Brick&gt;
       class Array;
       </programlisting>
       is a specification for creating a set of classes all named
*************** HERE</para>
*** 1150,1186 ****
      <section id="concepts-containers-field">
       <title>&field;</title>
  
!      <para>QUESTION: Do we include boundary conditions here?
  
       FINISH: Do we have an example that shows something not possible
       with &array;?</para>
  
!      <para>Describe and illustrate multi-material and
       multivalue?</para>
- 
-      <para>ADD: description of meshes and guard layers.</para>
  
!     </section>
! 
  
!    </section>
  
!    <section id="concepts-engines">
!     <title>Engines</title>
  
!     <para>(unformatted) From
      <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
  
!     <para>The Engine Concept</para>
  
!     <para>To implement polymorphic indexing, the Array class defers
      data storage and data lookup to an engine object. The requirements
      that the Array template places on its engine provide the
      definition for the Engine concept. We'll describe these by
      examining a simplified version of the Array template, shown in
      Fig. 2.</para>
  
!     <para>First, the Array class determines and exports (makes
      Engine_t part of Array's public interface) the type of the engine
      class that it will use:
      <programlisting>
--- 567,601 ----
      <section id="concepts-containers-field">
       <title>&field;</title>
  
! 	     <para>QUESTION: Do we include boundary conditions here?
  
       FINISH: Do we have an example that shows something not possible
       with &array;?</para>
  
! 	     <para>Describe and illustrate multi-material and
       multivalue?</para>
  
! 	     <para>ADD: description of meshes and guard layers.</para>
  
! 	    </section>
!      </section>
  
!      <section id="concepts-engines">
!       <title>Engines</title>
  
!       <para>(unformatted) From
      <filename>papers/GenericProgramming_CSE/dubois.html</filename>:</para>
  
!       <para>The Engine Concept</para>
  
!       <para>To implement polymorphic indexing, the Array class defers
      data storage and data lookup to an engine object. The requirements
      that the Array template places on its engine provide the
      definition for the Engine concept. We'll describe these by
      examining a simplified version of the Array template, shown in
      Fig. 2.</para>
  
!       <para>First, the Array class determines and exports (makes
      Engine_t part of Array's public interface) the type of the engine
      class that it will use:
      <programlisting>
*************** HERE</para>
*** 1215,1221 ****
      concept: it must provide a version of operator() that takes Dim
      values of type Index_t.</para>
  
!     <para>Simply passing the indices on to the engine object may seem
      odd. After all, engine(i,j) looks like we're just indexing another
      array. There are several advantages to this extra level of
      indirection. The Array class is as faithful a model of the Array
--- 630,636 ----
      concept: it must provide a version of operator() that takes Dim
      values of type Index_t.</para>
  
!       <para>Simply passing the indices on to the engine object may seem
      odd. After all, engine(i,j) looks like we're just indexing another
      array. There are several advantages to this extra level of
      indirection. The Array class is as faithful a model of the Array
*************** HERE</para>
*** 1386,1497 ****
      </variablelist>
     </section>
  
- 
-    <section id="concepts-relations">
-     <title>Relations</title>
- 
-     <para>UNFINISHED</para>
-    </section>
- 
- 
-    <section id="concepts-stencils">
-     <title>Stencils</title>
- 
-     <para>Section&nbsp;3.5.4, "Stencil Objects," of
-     <filename>papers/pooma.ps</filename> provides a few uses of
-     stencils.</para>
- 
-     <para>Section&nbsp;5, "Performance," of
-     <filename>papers/iscope98.pdf</filename> motivates and explains
-     stencils.</para>
-    </section>
- 
- 
-    <section id="concepts-contexts">
-     <title>Contexts</title>
- 
-     <blockquote>
-      <attribution><filename
-      class="libraryfile">background.html</filename></attribution>
-      <para>In order to be able to cope with the variations in machine
-      architecture noted above, &pooma;'s distributed execution model
-      is defined in terms of one or more contexts, each of which may
-      host one or more threads. A context is a distinct region of
-      memory in some computer. The threads associated with the context
-      can access data in that memory region and can run on the
-      processors associated with that context. Threads running in
-      different contexts cannot access memory in other contexts.</para>
- 
-      <para>A single context may include several physical processors,
-      or just one. Conversely, different contexts do not have to be on
-      separate computers&mdash;for example, a 32-node SMP computer could
-      have up to 32 separate contexts. This release of &pooma; only
-      supports a single context for each application, but can use
-      multiple threads in the context on supported platforms. Support
-      for multiple contexts will be added in an upcoming
-      release.</para>
-     </blockquote>
-    </section>
- 
-   </chapter>
- 
- 
-   <chapter id="sequential">
-    <title>Writing Sequential Programs</title>
- 
-    <para>&pooma; can reorder computations to permit more efficient
-    computation.  When running a sequential program, reordering may
-    permit omission of unneeded computations.  For example, if only
-    values from a particular field are printed, only computations
-    involving the field and containers dependent on it need to occur.
-    When running a distributed program, reordering may permit
-    computation and communication among processors to overlap.  &pooma;
-    automatically tracks dependences between data-parallel expressions,
-    ensuring correct ordering.  It does not track statements accessing
-    particular &array; and &field; values so the programmer must
-    precede these statements with calls to
-    <function>Pooma::blockAndEvaluate()</function>.  Each call forces
-    the executable to wait until all computation has completed.  Thus,
-    the desired values are known to be available.  In practice, some
-    calls to <function>Pooma::blockAndEvaluate</function> may not be
-    necessary, but omitting them requires knowledge of &pooma;'s
-    dependence computations, so the &author; recommends calling
-    <function>Pooma::blockAndEvaluate</function> before each access to
-    a particular value in an &array; or &field;.  Omitting a necessary
-    call may lead to a race condition.  See <xref
-    linkend="debugging_profiling-missing_blockandevaluate"></xref> for
-    instructions how to diagnose and eliminate these race conditions.</para>
- 
-    <para>Section&nbsp;3, "Domains and Views," of
-    <filename>papers/iscope98.pdf</filename> describes five types of
-    domains.</para>
- 
-    <para>UNFINISHED</para>
- 
-    <section id="sequential-benchmarks">
-     <title>&benchmark; Programs</title>
- 
-     <para>Define a &benchmark; program vs. an example or an
-     executable.  Provide a short overview of how to run these
-     programs.  Provide an overview of how to write these programs.
-     See <filename
-     class="headerfile">src/Utilities/Benchmark.h</filename>.</para>
-    </section>
- 
- 
-    <section id="sequential-inform">
-     <title>Using <type>Inform</type>s for Output</title>
- 
-     <para>UNFINISHED</para>
-    </section>
- 
- 
-    <section>
-     <title>Miscellaneous</title>
- 
-    <para>Section&nbsp;3, "Domains and Views," of
-     <filename>papers/iscope98.pdf</filename> describes five types of
-     domains.</para>
     </section>
    </chapter>
  
--- 801,806 ----
*************** HERE</para>
*** 3579,3588 ****
     <para>To use multiple processors with &pooma; requires installing
     the &cheetah; messaging library and an underlying messaging library
     such as the Message Passing Interface (&mpi;) Communications
!    Library or the &mm; Shared Memory Library.  In this section, we
!    first describe how to install &mm;.  Read the section only if using
!    &mm;, not &mpi;.  Then we describe how to install &cheetah; and
!    configure &pooma; to use it.</para>
  
     <section id="installation-distributed_computing-mm">
      <title>Obtaining and Installing the &mm; Shared Memory Library</title>
--- 2888,2897 ----
     <para>To use multiple processors with &pooma; requires installing
     the &cheetah; messaging library and an underlying messaging library
     such as the Message Passing Interface (&mpi;) Communications
!    Library or the &mm; Shared Memory Library.  In the following
!    section, we first describe how to install &mm;.  Read it only if
!    using &mm;, not &mpi;.  Then we describe how to install &cheetah;
!    and configure &pooma; to use it.</para>
  
     <section id="installation-distributed_computing-mm">
      <title>Obtaining and Installing the &mm; Shared Memory Library</title>
*************** HERE</para>
*** 3834,3839 ****
--- 3143,3154 ----
     <title>Miscellaneous</title>
  
     <orderedlist>
+     <listitem>
+      <para>Section&nbsp;5, "Performance," of
+      <filename>papers/iscope98.pdf</filename> motivates and explains
+      stencils.</para>
+     </listitem>
+ 
      <listitem>
       <para>If there is time, present another example program, e.g., a
       Jacobi solver.</para>
Index: figures/concepts.mp
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/figures/concepts.mp,v
retrieving revision 1.1
diff -c -p -r1.1 concepts.mp
*** figures/concepts.mp	2001/12/11 20:36:13	1.1
--- figures/concepts.mp	2001/12/13 04:02:07
*************** beginfig(101)
*** 144,150 ****
    boxit.l6(btex \strut \type{Field}: etex);
    boxit.l7(btex \strut $\mbox{index} \mapsto \mbox{value}$ etex);
    boxit.l9(btex \strut \type{Field}: etex);
!   boxit.l10(btex \strut $\mbox{indices} \mapsto \mbox{geometric value}$ etex);
    fixsize(l1,l2,l3,l4,l6,l7,l9,l10);
    
    ypart(l1.c - l2.c) = 0;
--- 144,150 ----
    boxit.l6(btex \strut \type{Field}: etex);
    boxit.l7(btex \strut $\mbox{index} \mapsto \mbox{value}$ etex);
    boxit.l9(btex \strut \type{Field}: etex);
!   boxit.l10(btex \strut $\mbox{indices} \mapsto \mbox{spatial value}$ etex);
    fixsize(l1,l2,l3,l4,l6,l7,l9,l10);
    
    ypart(l1.c - l2.c) = 0;
*************** beginfig(101)
*** 163,169 ****
    % Create and layout the mesh boxes.
    boxit.ia[2](btex indices etex);
    boxit.ea[2](btex mesh etex);
!   boxit.va[2](btex geometric value etex);
    fixsize(ia[2],ea[2],va[2]);
    ia[1].w - ia[2].w = 0.6(ia[0].w - ia[1].w);
    ypart(va[2].w - ea[2].e) = ypart(ea[2].w - ia[2].e) = 0;
--- 163,169 ----
    % Create and layout the mesh boxes.
    boxit.ia[2](btex indices etex);
    boxit.ea[2](btex mesh etex);
!   boxit.va[2](btex spatial value etex);
    fixsize(ia[2],ea[2],va[2]);
    ia[1].w - ia[2].w = 0.6(ia[0].w - ia[1].w);
    ypart(va[2].w - ea[2].e) = ypart(ea[2].w - ia[2].e) = 0;

From oldham at codesourcery.com  Fri Dec 14 05:19:26 2001
From: oldham at codesourcery.com (Jeffrey Oldham)
Date: Thu, 13 Dec 2001 21:19:26 -0800
Subject: Manual Patch: Some Concepts Changes 
Message-ID: <20011213211926.A29012@codesourcery.com>

2001-Dec-13  Jeffrey D. Oldham  <oldham at codesourcery.com>

	These changes mainly represent wordsmithing of the concepts
	chapter and some preliminary work on the "Writing Sequential
	Programs" chapter.

	* concepts.xml: Wordsmithing and a little rearrangement.
	* glossary.xml (interval): Improve wording.
	* makefile (manual.dvi): Improve dependence information.
	* manual.xml: Add planning material for the "Writing Sequential
	Programs" chapter.
	* tutorial.xml: Fix an article.
	* figures/concepts.mp: Shrink the figure's horizontal extent.

Applied to	mainline

Jeffrey D. Oldham
oldham at codesourcery.com
-------------- next part --------------
Index: concepts.xml
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/concepts.xml,v
retrieving revision 1.1
diff -c -p -r1.1 concepts.xml
*** concepts.xml	2001/12/13 04:04:05	1.1
--- concepts.xml	2001/12/14 04:12:54
***************
*** 6,20 ****
  
   <para>In the previous chapter, we presented several different
   implementations of the &doof2d; simulation program.  The
!  implementations illustrate the various containers, computation
!  modes, and computation environments that &pooma; supports.  In this
!  chapter, we describe the concepts associated with each of these
!  three categories.  Specific details needed by programmers are
!  deferred to later chapters.</para>
  
   <table frame="none" colsep="0" rowsep="0" tocentry="1"
     orient="port" pgwide="0" id="concepts-table">
!    <title>&pooma; Implementation Concepts</title>
     <tgroup cols="3" align="center">
      <thead>
       <row>
--- 6,55 ----
  
   <para>In the previous chapter, we presented several different
   implementations of the &doof2d; simulation program.  The
!  implementations illustrate the various containers, computation modes,
!  and computation environments that &pooma; supports.  In this chapter,
!  we describe the concepts associated with each of these three
!  categories.  Specific details needed for their use are deferred to
!  later chapters.</para>
  
+  <para>The most important &pooma; concepts can be grouped into three
+  separate categories:
+  <variablelist>
+   <varlistentry>
+     <term>container</term>
+     <listitem>
+      <para>data structure holding one or more values and addressed
+      by indices</para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>computation modes</term>
+     <listitem>
+      <para>styles of expressing computations and accesses to container
+      values</para>
+     </listitem>
+    </varlistentry>
+    <varlistentry>
+     <term>computation environment</term>
+     <listitem>
+      <para>description of resources for computing, e.g., single
+      processor or multi-processor.</para>
+     </listitem>
+    </varlistentry>
+   </variablelist>
+   See <xref linkend="concepts-table"></xref>.  Many &pooma; programs
+   select one possibility from each column.  For example, <xref
+   linkend="tutorial-array_stencil-doof2d"></xref> used &array;
+   containers and stencils for sequential computation, while <xref
+   linkend="tutorial-field_distributed-doof2d"></xref> used &field;
+   containers and data-parallel statements with distributed
+   computation.  A program may use multiple containers and various
+   computation modes, but the computation environment is either
+   distributed or not.</para>
+ 
   <table frame="none" colsep="0" rowsep="0" tocentry="1"
     orient="port" pgwide="0" id="concepts-table">
!    <title>&pooma; Concepts</title>
     <tgroup cols="3" align="center">
      <thead>
       <row>
***************
*** 57,134 ****
      </tbody>
     </tgroup>
    </table>
- 
-  <para>The most important &pooma; concepts can be grouped into three
-  separate categories:
-  <variablelist>
-   <varlistentry>
-     <term>container</term>
-     <listitem>
-      <para>data structure holding one or more values and addressed
-      by indices</para>
-     </listitem>
-    </varlistentry>
-    <varlistentry>
-     <term>computation modes</term>
-     <listitem>
-      <para>styles of expressing computations</para>
-     </listitem>
-    </varlistentry>
-    <varlistentry>
-     <term>computation environment</term>
-     <listitem>
-      <para>description of resources for computing, e.g., single
-      processor or multi-processor</para>
-     </listitem>
-    </varlistentry>
-   </variablelist>
-   See <xref linkend="concepts-table"></xref>.  Many &pooma; programs
-   select one possibility from each column.  For example, <xref
-   linkend="tutorial-array_stencil-doof2d"></xref> used a &array;
-   container and stencils for sequential computation, while <xref
-   linkend="tutorial-field_distributed-doof2d"></xref> used a &field;
-   container and data-parallel statements with distributed
-   computation.  A program may use multiple containers and various
-   computation modes, but the computation environment either has
-   distributed processors or not.</para>
  
!  <para>In the rest of this chapter, we explore these three
!  categories.  First, we describe &pooma; containers, illustrating
!  the purposes of each, and explaining the concepts needed to declare
!  them.  Then, we describe the different computation modes and
!  finally distributed computation concepts.</para>
  
  
   <section id="concepts-containers">
    <title>&pooma; Containers</title>
  
!   <para>Most &pooma; programs use <firstterm>containers</firstterm>
!   to store groups of values.  &pooma; containers are objects that
!   store other objects.  They control allocation and deallocation of
!   and access to these objects.  They are a generalization of &c;
!   arrays, but &pooma; containers are first-class objects so they can
!   be used directly in expressions.  They are similar to &cc;
!   containers such as <type>vector</type>, <type>list</type>, and
!   <type>stack</type>.  See <xref
    linkend="concepts-containers-table"></xref> for a summary of the
    containers.</para>
  
!   <para>This chapter describes many concepts, not all of which are
!   needed to begin programming with the &pooma; Toolkit.  Below we
!   introduce the different categories of concepts.  After that, we
!   introduce the different &pooma;'s containers and describe how to
!   choose the appropriate one for a particular task.  <xref
    linkend="concepts-sequential_containers-declarations-dependences"></xref>
    indicates which concepts must be understood when declaring a
!   particular container.  All of these concepts are described in
!   <xref
    linkend="concepts-containers-sequential_declarations"></xref> and
    <xref
!   linkend="concepts-containers-distributed_declarations"></xref>.
!   Use this figure to decide which concepts in the former are
!   relevant.  Reading the latter section is necessary only if
!   computing using multiple processors.  The programs in the previous
!   chapter illustrate many of these concepts.</para>
  
    <table frame="none" colsep="0" rowsep="0" tocentry="1"
  	   orient="port" pgwide="0" id="concepts-containers-table">
--- 92,133 ----
      </tbody>
     </tgroup>
    </table>
  
!  <para>In the rest of this chapter, we explore these three categories.
!  First, we describe &pooma; containers, illustrating the purposes of
!  each, and explaining the concepts needed to declare them.  Then, we
!  describe the different computation modes and distributed computation
!  concepts.</para>
  
  
   <section id="concepts-containers">
    <title>&pooma; Containers</title>
  
!   <para>Most &pooma; programs use <firstterm>containers</firstterm> to
!   store groups of values.  &pooma; containers are objects that store
!   other objects such as numbers or vectors.  They control allocation
!   and deallocation of and access to these stored objects.  They are a
!   generalization of &c; arrays, but &pooma; containers are first-class
!   objects so they can be used directly in expressions.  They are
!   similar to &cc; containers such as <type>vector</type>,
!   <type>list</type>, and <type>stack</type>.  See <xref
    linkend="concepts-containers-table"></xref> for a summary of the
    containers.</para>
  
!   <para>This section describes many concepts, but one need not
!   understand them all to begin programming with the &pooma; Toolkit.
!   First, we introduce the different &pooma;'s containers and describe
!   how to choose an appropriate one for a particular task.  <xref
    linkend="concepts-sequential_containers-declarations-dependences"></xref>
    indicates which concepts must be understood when declaring a
!   particular container.  All of these concepts are described in <xref
    linkend="concepts-containers-sequential_declarations"></xref> and
    <xref
!   linkend="concepts-containers-distributed_declarations"></xref>.  Use
!   this figure to decide which concepts in the former are relevant.
!   Reading the latter section is necessary only if computing using
!   multiple processors.  The programs in the previous chapter
!   illustrate many of these concepts.</para>
  
    <table frame="none" colsep="0" rowsep="0" tocentry="1"
  	   orient="port" pgwide="0" id="concepts-containers-table">
***************
*** 175,191 ****
  <!-- FIXME: Want firstterm around array. -->
  
    <para>A &pooma; <glossterm
!   linkend="glossary-array">array;</glossterm>, generalizing a &c;
!   array, maps indices to values.  Given a index or position in an
    &array;'s domain, it returns the associated value, either by
    returning a stored value or by computing it.  The use of indices,
    which are usually ordered tuples, permits constant-time access
!   although computing a particular value may require significant
!   time.  In addition to the functionality provided by &c; arrays,
!   the &array; class automatically handles memory allocation and
!   deallocation, supports a wider variety of assignments, and can be
!   used in expressions.  For example, the addition of two arrays can
!   be assigned to an array and the product of a scalar element and an
    array is permissible.</para>
  
  <!-- FIXME: Want firstterm around dynamicarray. -->
--- 174,190 ----
  <!-- FIXME: Want firstterm around array. -->
  
    <para>A &pooma; <glossterm
!   linkend="glossary-array">&array;</glossterm> generalizes a &c; array
!   and maps indices to values.  Given an index or position in an
    &array;'s domain, it returns the associated value, either by
    returning a stored value or by computing it.  The use of indices,
    which are usually ordered tuples, permits constant-time access
!   although computing a particular value may require significant time.
!   In addition to the functionality provided by &c; arrays, the &array;
!   class automatically handles memory allocation and deallocation,
!   supports a wider variety of assignments, and can be used in
!   expressions.  For example, the addition of two arrays can be
!   assigned to an array and the product of a scalar element and an
    array is permissible.</para>
  
  <!-- FIXME: Want firstterm around dynamicarray. -->
***************
*** 200,215 ****
  
    <para>A &pooma; <glossterm>&field;</glossterm> is an &array; with
    spatial extent.  Each domain consists of <glossterm
!   linkend="glossary-cell"><firstterm>cell</firstterm></glossterm>s
!   in one-, two-, or three-dimensional space.  Although indexed
!   similarly to &array;s, each cell may contain multiple values and
!   multiple materials.  A &field;'s <glossterm
    linkend="glossary-mesh">mesh</glossterm> stores its spatial
!   characteristics and can map yield, e.g., a point contained in a
!   cell, the distance between two cells, and a cell's normals.  A
    &field; should be used whenever geometric or spatial computations
!   are needed, multiple values per index are desired, or a
!   computation involves more than one material.</para>
  
  <!-- FIXME: Want firstterm around tensor. -->
  
--- 199,214 ----
  
    <para>A &pooma; <glossterm>&field;</glossterm> is an &array; with
    spatial extent.  Each domain consists of <glossterm
!   linkend="glossary-cell"><firstterm>cell</firstterm></glossterm>s in
!   one-, two-, or three-dimensional space.  Although indexed similarly
!   to &array;s, each cell may contain multiple values and multiple
!   materials.  A &field;'s <glossterm
    linkend="glossary-mesh">mesh</glossterm> stores its spatial
!   characteristics and can map yield, e.g., the cell at a particular
!   point, the distance between two cells, or a cell's normals.  A
    &field; should be used whenever geometric or spatial computations
!   are needed, multiple values per index are desired, or a computation
!   involves more than one material.</para>
  
  <!-- FIXME: Want firstterm around tensor. -->
  
***************
*** 222,229 ****
  
    <para>A <glossterm linkend="glossary-matrix">&matrix;</glossterm>
    implements a two-dimensional mathematical matrix.  Since it is a
!   first-class type, it can be used in expressions such as
!   multiplying matrices and assignments to matrices.</para>
  
  <!-- FIXME: Want firstterm around vector. -->
  
--- 221,228 ----
  
    <para>A <glossterm linkend="glossary-matrix">&matrix;</glossterm>
    implements a two-dimensional mathematical matrix.  Since it is a
!   first-class type, it can be used in expressions such as assignments
!   to matrices and multiplying matrices.</para>
  
  <!-- FIXME: Want firstterm around vector. -->
  
***************
*** 234,259 ****
    multiplying a &matrix; and a &vector;.</para>
  
    <para>The data of an &array;, &dynamicarray;, or &field; can be
!   viewed using more than one container by taking a view.  A
!   <glossterm
    linkend="glossary-view"><firstterm>view</firstterm></glossterm> of
    an existing container&nbsp;&container; is a container whose domain
!   is a subset of&nbsp;&container;.  The subset can equal the
!   original domain.  A view acts like a reference in that changing
!   any of the view's values also changes the original container's and
!   vice versa.  While users sometimes explicitly create views, they
!   are perhaps more frequently created as temporaries in expressions.
!   For example, if <varname>A</varname> is an &array; and
!   <varname>I</varname> is a domain, <statement>A(I) -
!   A(I-1)</statement> forms the difference between adjacent
!   values.</para>
  
  
    <section id="concepts-containers-choosing">
     <title>Choosing a Container</title>
  
     <para>The two most commonly used &pooma; containers are &array;s
!    and &field;s.  <xref
     linkend="concepts-containers-choice_table"></xref> contains a
     decision tree describing how to choose an appropriate
     container.</para>
--- 233,257 ----
    multiplying a &matrix; and a &vector;.</para>
  
    <para>The data of an &array;, &dynamicarray;, or &field; can be
!   viewed using more than one container by taking a view.  A <glossterm
    linkend="glossary-view"><firstterm>view</firstterm></glossterm> of
    an existing container&nbsp;&container; is a container whose domain
!   is a subset of&nbsp;&container;.  The subset can equal the original
!   domain.  A view acts like a reference in that changing any of the
!   view's values also changes the original container's and vice versa.
!   While users sometimes explicitly create views, they are perhaps more
!   frequently created as temporaries in expressions.  For example, if
!   <varname>A</varname> is an &array; and <varname>I</varname> is a
!   domain, <statement>A(I) - A(I-1)</statement> uses two views to form
!   the difference between adjacent values.</para>
  
  
    <section id="concepts-containers-choosing">
     <title>Choosing a Container</title>
  
     <para>The two most commonly used &pooma; containers are &array;s
!    and &field;s, while &vector;, &matrix;, or &tensor; frequently
!    represent mathematical objects.  <xref
     linkend="concepts-containers-choice_table"></xref> contains a
     decision tree describing how to choose an appropriate
     container.</para>
***************
*** 298,303 ****
--- 296,307 ----
    <section id="concepts-containers-sequential_declarations">
     <title>Declaring Sequential Containers</title>
  
+    <para>In the previous sections, we introduced the &pooma;
+    containers and described how to choose one appropriate for a
+    given task.  In this section, we describe the concepts involved
+    in declaring them.  Concepts specific to distributed computation
+    are described in the next section.</para>
+ 
     <figure float="1" id="concepts-sequential_containers-declarations-dependences">
      <title>Concepts For Declaring Containers</title>
      <mediaobject>
***************
*** 310,347 ****
      </mediaobject>
     </figure>
  
-    <para>In the previous sections, we introduced the &pooma;
-    containers and described how to choose one appropriate for a
-    given task.  In this section, we describe the concepts involved
-    in declaring them.  Concepts specific to distributed computation
-    are described in the next section.</para>
- 
     <para><xref
     linkend="concepts-sequential_containers-declarations-dependences"></xref>
     illustrates the containers and the concepts involved in their
     declarations.  The containers are listed in the top row.  Lines
     connect these containers to the components necessary for their
     declarations.  For example, an &array; declaration requires an
!    engine and a layout.  These, in turn, depend on other &pooma;
     concepts.  Declarations necessary only for distributed, or
!    multiprocessor, computation are surrounded by dashed lines.  You
!    can use these dependences to indicate the concepts needed for a
!    particular container.</para>
  
     <para>An <glossterm
     linkend="glossary-engine"><firstterm>engine</firstterm></glossterm>
     stores and, if necessary, computes a container's values.  A
     container has one or more engines.  The separation of a container
!    and its storage permits optimizing a program's space
     requirements.  For example, a container returning the same value
     for all indices can use a constant engine, which need only store
     one value for the entire domain.  A &compressiblebrick; engine
     reduces its space requirements to a constant whenever all its
     values are the same.  The separation also permits taking <link
!    linkend="glossary-view">view</link>s of containers without
!    copying storage.</para>
  
!    <figure float="1" id="concepts-containers-declarations-computational_implementation">
      <title>&array; and &field; Mathematical and Computational Concepts</title>
      <mediaobject>
       <imageobject>
--- 314,345 ----
      </mediaobject>
     </figure>
  
     <para><xref
     linkend="concepts-sequential_containers-declarations-dependences"></xref>
     illustrates the containers and the concepts involved in their
     declarations.  The containers are listed in the top row.  Lines
     connect these containers to the components necessary for their
     declarations.  For example, an &array; declaration requires an
!    engine and a layout.  These, in turn, can depend on other &pooma;
     concepts.  Declarations necessary only for distributed, or
!    multiprocessor, computation are surrounded by dashed lines.  These
!    dependences to indicate the concepts needed for a particular
!    container.</para>
  
     <para>An <glossterm
     linkend="glossary-engine"><firstterm>engine</firstterm></glossterm>
     stores and, if necessary, computes a container's values.  A
     container has one or more engines.  The separation of a container
!    from its storage permits optimizing a program's space and time
     requirements.  For example, a container returning the same value
     for all indices can use a constant engine, which need only store
     one value for the entire domain.  A &compressiblebrick; engine
     reduces its space requirements to a constant whenever all its
     values are the same.  The separation also permits taking <link
!    linkend="glossary-view">view</link>s of containers without copying
!    storage.</para>
  
!    <figure float="1" pgwide="1" id="concepts-containers-declarations-computational_implementation">
      <title>&array; and &field; Mathematical and Computational Concepts</title>
      <mediaobject>
       <imageobject>
***************
*** 356,387 ****
     <para>A <glossterm
     linkend="glossary-layout"><firstterm>layout</firstterm></glossterm>
     maps <link linkend="glossary-domain">domain</link> indices to the
!    processors and computer memory used by a container's engines.
!    See <xref
     linkend="concepts-containers-declarations-computational_implementation"></xref>.
!    A computer computes a container's values using a processor and
!    memory.  The layout specifies the processor(s) and memory to use
!    for each particular index.  A container's layout for a
!    uniprocessor implementation consists of its domain, the
!    processor, and its memory.  For a multi-processor implementation,
!    the layout maps portions of the domain to (possibly different)
!    processors and memory.</para>
  
     <para>A &field;'s <glossterm
     linkend="glossary-mesh"><firstterm>mesh</firstterm></glossterm>
     maps <link linkend="glossary-domain">domain</link> indices to
!    spatial values in &space; such as distance between cells, edge
     lengths, and normals to cells.  In other words, it provides a
     &field;'s spatial extent.  See also <xref
     linkend="concepts-containers-declarations-computational_implementation"></xref>.
!    Different mesh types may support different spatial
!    values.</para>
  
     <para>A mesh's <glossterm
     linkend="glossary-corner_position"><firstterm>corner
     position</firstterm></glossterm> specifies the <link
     linkend="glossary-point">point</link> in &space; corresponding to
!    the lower, left corner of its <link
     linkend="glossary-domain">domain</link>.  Combining this, the
     domain, and the cell size fully specifies the mesh's map from
     indices to &space;.</para>
--- 354,408 ----
     <para>A <glossterm
     linkend="glossary-layout"><firstterm>layout</firstterm></glossterm>
     maps <link linkend="glossary-domain">domain</link> indices to the
!    processors and computer memory used by a container's engines.  See
!    <xref
     linkend="concepts-containers-declarations-computational_implementation"></xref>.
!    A program computes a container's values using a processor and
!    memory.  The layout specifies the processors and memory to use for
!    each particular index.  A container's layout for a uniprocessor
!    implementation consists of its domain, the processor, and its
!    memory.  For a multi-processor implementation, the layout maps
!    portions of the domain to (possibly different) processors and
!    memory.</para>
  
+    <para>A <glossterm
+    linkend="glossary-domain"><firstterm>domain</firstterm></glossterm>
+    is a set of points on which a container can define values.  There
+    are several different types of domains.  An <glossterm
+    linkend="glossary-interval"><firstterm>interval</firstterm></glossterm>
+    consists of all integral points between two endpoints.  It is
+    frequently represented using mathematical interval notation [a,b]
+    even though it contains only the integral points, e.g., a, a+1,
+    a+2, &hellip;, b.  The concept is generalized to multiple
+    dimensions by forming tensor product of intervals, i.e., all the
+    integral tuples in an &n;-dimensional space.  For example, the
+    two-dimensional containers in the previous chapter are defined on a
+    two-dimensional domain with the both dimensions' spanning the
+    interval [0,n).  A domain need not contain all integral points
+    between its endpoints.  A <glossterm
+    linkend="glossary-stride"><firstterm>stride</firstterm></glossterm>
+    is a subset of an interval consisting of regularly-spaced points.
+    A <glossterm
+    linkend="glossary-range"><firstterm>range</firstterm></glossterm>
+    is a subset of an interval formed by the tensor product of strides.
+    A <glossterm
+    linkend="glossary-region"><firstterm>region</firstterm></glossterm>
+    represents a continuous &n;-dimensional domain.</para>
+ 
     <para>A &field;'s <glossterm
     linkend="glossary-mesh"><firstterm>mesh</firstterm></glossterm>
     maps <link linkend="glossary-domain">domain</link> indices to
!    spatial values in &space; such as distances between cells, edge
     lengths, and normals to cells.  In other words, it provides a
     &field;'s spatial extent.  See also <xref
     linkend="concepts-containers-declarations-computational_implementation"></xref>.
!    Different mesh types may support different spatial values.</para>
  
     <para>A mesh's <glossterm
     linkend="glossary-corner_position"><firstterm>corner
     position</firstterm></glossterm> specifies the <link
     linkend="glossary-point">point</link> in &space; corresponding to
!    the cell in the lower, left corner of its <link
     linkend="glossary-domain">domain</link>.  Combining this, the
     domain, and the cell size fully specifies the mesh's map from
     indices to &space;.</para>
***************
*** 393,434 ****
     width, height, and depth, in&nbsp;&space;.  Combining this, the
     domain, and the corner position fully specifies the mesh's map
     from indices to &space;.</para>
- 
-    <para>A <glossterm
-    linkend="glossary-domain"><firstterm>domain</firstterm></glossterm>
-    is a set of points on which a container can define values.  An
-    <glossterm
-    linkend="glossary-interval"><firstterm>interval</firstterm></glossterm>
-    consists of all integral points between two values.  It is
-    frequently represented using mathematical interval notation [a,b]
-    even though it contains only the integral points, e.g., a, a+1,
-    a+2, &hellip;, b.  The concept is generalized to multiple
-    dimensions by forming tensor product of intervals, i.e., all the
-    integral tuples in an &n;-dimensional space.  For example, the
-    two-dimensional containers in the previous chapter are defined on
-    a two-dimensional domain with the both dimensions' spanning the
-    interval [0,n).  A <glossterm
-    linkend="glossary-stride"><firstterm>stride</firstterm></glossterm>
-    is a subset of an interval consisting of regularly-spaced
-    points.  A <glossterm
-    linkend="glossary-range"><firstterm>range</firstterm></glossterm>
-    is a subset of an interval formed by the tensor product of strides.
-    A <glossterm
-    linkend="glossary-region"><firstterm>region</firstterm></glossterm>
-    represents a continuous &n;-dimensional domain.</para>
    </section>
  
  
    <section id="concepts-containers-distributed_declarations">
     <title>Declaring Distributed Containers</title>
  
!    <para>In the previous section, we introduced the concepts
!    important when declaring containers for use on uniprocessor
!    computers.  When using multi-processor computers, we augment
!    these concepts with those for distributed computation.  Reading
!    this section is important only for running the same program on
!    multiple processors.  Many of these concepts were introduced in
!    <xref linkend="tutorial-array_distributed"></xref> and <xref
     linkend="tutorial-field_distributed"></xref>.  <xref
     linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
     illustrates the &pooma; distributed computation model.  In this
--- 414,432 ----
     width, height, and depth, in&nbsp;&space;.  Combining this, the
     domain, and the corner position fully specifies the mesh's map
     from indices to &space;.</para>
    </section>
  
  
    <section id="concepts-containers-distributed_declarations">
     <title>Declaring Distributed Containers</title>
  
!    <para>In the previous section, we introduced the concepts important
!    when declaring containers for use on uniprocessor computers.  When
!    using multi-processor computers, we augment these concepts with
!    those for distributed computation.  Reading this section is
!    important only for running a program on multiple processors.  Many
!    of these concepts were introduced in <xref
!    linkend="tutorial-array_distributed"></xref> and <xref
     linkend="tutorial-field_distributed"></xref>.  <xref
     linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
     illustrates the &pooma; distributed computation model.  In this
***************
*** 447,480 ****
  
     <para>A <glossterm
     linkend="glossary-partition"><firstterm>partition</firstterm></glossterm>
!    specified how to divide a container's domain into distributed
     pieces.  For example, the partition illustrated in <xref
     linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
     would divide a two-dimensional domain into three equally-sized
!    pieces along the x-dimension and two equally-sized pieces along
!    the y-dimension.  Partitions can be independent of the size of
!    container's domain.  The example partition will work on any
!    domain as long as the size of its x-dimension is a multiple of
!    three.  A domain is separated into disjoint <link
     linkend="glossary-patch">patch</link>es.</para>
  
!    <para>A <glossterm
!    linkend="glossary-guard_layer"><firstterm>guard
!    layer</firstterm></glossterm> is <quote>extra</quote> domain
!    surrounding each patch.  This region has read-only values.  An
!    <glossterm
     linkend="glossary-external_guard_layer"><firstterm>external guard
     layer</firstterm></glossterm> specifies values surrounding the
!    domain.  Its presence eases computation along the domain's edges
!    by permitting the same computations as for more internal
     computations.  An <glossterm
     linkend="glossary-internal_guard_layer"><firstterm>internal guard
     layer</firstterm></glossterm> duplicates values from adjacent
     patches so communication with adjacent patches need not occur
     during a patch's computation.  The use of guard layers is an
     optimization; using external guard layers eases programming and
!    using internal guard layers reduces communication among
!    processors.  Their use is not required.</para>
  
     <para>A <glossterm
     linkend="glossary-context_mapper"><firstterm>context
--- 445,476 ----
  
     <para>A <glossterm
     linkend="glossary-partition"><firstterm>partition</firstterm></glossterm>
!    specifies how to divide a container's domain into distributed
     pieces.  For example, the partition illustrated in <xref
     linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
     would divide a two-dimensional domain into three equally-sized
!    pieces along the x-dimension and two equally-sized pieces along the
!    y-dimension.  Partitions can be independent of the size of
!    container's domain.  The example partition will work on any domain
!    as long as the size of its x-dimension is a multiple of three.  A
!    domain is separated into disjoint <link
     linkend="glossary-patch">patch</link>es.</para>
  
!    <para>A <glossterm linkend="glossary-guard_layer"><firstterm>guard
!    layer</firstterm></glossterm> surrounds each patch with read-only
!    values.  An <glossterm
     linkend="glossary-external_guard_layer"><firstterm>external guard
     layer</firstterm></glossterm> specifies values surrounding the
!    entire domain.  Its presence eases computation along the domain's
!    edges by permitting the same computations as for more internal
     computations.  An <glossterm
     linkend="glossary-internal_guard_layer"><firstterm>internal guard
     layer</firstterm></glossterm> duplicates values from adjacent
     patches so communication with adjacent patches need not occur
     during a patch's computation.  The use of guard layers is an
     optimization; using external guard layers eases programming and
!    using internal guard layers reduces communication among processors.
!    Their use is not required.</para>
  
     <para>A <glossterm
     linkend="glossary-context_mapper"><firstterm>context
***************
*** 496,512 ****
    <title>Computation Modes</title>
  
    <para>&pooma; computations can be expressed using a variety of
!   modes.  Most of &pooma; computations involve &array; or &field;
    containers, but how their values are accessed and the associated
!   algorithms using them varies.  Element-wise computation involves
!   explicitly accessing values.  A data-parallel computation uses
!   expressions to represent larger subsets of a container's values.
!   Stencil-based computations write a computation as repeatedly
!   applying a local computation to each element of an array.  A
!   relation among containers establishes a dependency between them so
!   the values of one container are updated whenever any other's
!   values change.  A program may use any or all of these styles,
!   described below.</para>
  
    <para><glossterm
    linkend="glossary-element_wise"><firstterm>Element-wise</firstterm></glossterm>
--- 492,508 ----
    <title>Computation Modes</title>
  
    <para>&pooma; computations can be expressed using a variety of
!   modes.  Many &pooma; computations involve &array; or &field;
    containers, but how their values are accessed and the associated
!   algorithms using them varies.  For example, element-wise computation
!   involves explicitly accessing a container's values.  A data-parallel
!   computation uses expressions to represent larger subsets of a
!   container's values.  Stencil-based computations express a
!   computation as repeatedly applying a local computation to each
!   element of an array.  A relation among containers establishes a
!   dependency among them so the values of one container are updated
!   whenever any other's values change.  A program may use any or all of
!   these styles, which are described below.</para>
  
    <para><glossterm
    linkend="glossary-element_wise"><firstterm>Element-wise</firstterm></glossterm>
***************
*** 515,521 ****
    container&nbsp;&container; might be referenced as
    <statement>&container(3,4)</statement> or
    <statement>&container(i,j+1)</statement>.  This is the usual
!   notation for languages without objects such as &c;.</para>
  
    <para><glossterm
    linkend="glossary-data_parallel"><firstterm>Data-parallel</firstterm></glossterm>
--- 511,517 ----
    container&nbsp;&container; might be referenced as
    <statement>&container(3,4)</statement> or
    <statement>&container(i,j+1)</statement>.  This is the usual
!   notation for non-object-oriented languages such as &c;.</para>
  
    <para><glossterm
    linkend="glossary-data_parallel"><firstterm>Data-parallel</firstterm></glossterm>
***************
*** 523,529 ****
    values.  For example, in <xref
    linkend="tutorial-array_parallel-doof2d"></xref>,
    <statement>a(I,J)</statement> represents the subset of &array;
!   <varname>a</varname>'s values with coordinates in the domain
    specified by the one-dimensional &interval;s <varname>I</varname>
    and <varname>J</varname>.  Using data-parallel expressions
    frequently eliminates the need for writing explicit loops in
--- 519,525 ----
    values.  For example, in <xref
    linkend="tutorial-array_parallel-doof2d"></xref>,
    <statement>a(I,J)</statement> represents the subset of &array;
!   <varname>a</varname>'s values having coordinates in the domain
    specified by the one-dimensional &interval;s <varname>I</varname>
    and <varname>J</varname>.  Using data-parallel expressions
    frequently eliminates the need for writing explicit loops in
***************
*** 532,555 ****
    <para>A <glossterm
    linkend="glossary-stencil"><firstterm>stencil</firstterm></glossterm>
    computes a container's value using neighboring data values.  Each
!   stencil consists of an indication of which neighboring values to
    read and a function using those values.  For example, an averaging
!   stencil may access all neighbors, averaging them.  In &pooma;, we
!   represent a stencil using a function object having functions
!   indicating which neighboring values are used.  Stencil
!   computations are frequently used in solving partial differential
!   equations, image processing, and geometric modeling.</para>
  
    <para>A <glossterm
    linkend="glossary-relation"><firstterm>relation</firstterm></glossterm>
!   is a dependence among containers so the dependent container's
!   values are updated when its values are needed and any of its
!   related containers' values have changed.  A relation is specified
!   by a dependent container, independent containers, and a function
    computing the dependent container's values using the independent
    containers' values.  To avoid excess computation, the dependent
!   container's values are computed only when needed, e.g., for
!   printing or for computing the values of another dependent
    container.  Thus, this computation is sometimes called <quote>lazy
    evaluation</quote>.</para>
   </section>
--- 528,552 ----
    <para>A <glossterm
    linkend="glossary-stencil"><firstterm>stencil</firstterm></glossterm>
    computes a container's value using neighboring data values.  Each
!   stencil consists of a specification of which neighboring values to
    read and a function using those values.  For example, an averaging
!   stencil may access all its adjacent neighbors, averaging them.  In
!   &pooma;, we represent a stencil using a function object with
!   additional functions indicating which neighboring values are used.
!   Stencil computations are frequently used in solving partial
!   differential equations, image processing, and geometric
!   modeling.</para>
  
    <para>A <glossterm
    linkend="glossary-relation"><firstterm>relation</firstterm></glossterm>
!   is a dependence among containers such the dependent container's
!   values are updated when its values are needed and any of its related
!   containers' values have changed.  A relation is specified by a
!   dependent container, independent containers, and a function
    computing the dependent container's values using the independent
    containers' values.  To avoid excess computation, the dependent
!   container's values are computed only when needed, e.g., for printing
!   the container or for computing the values of another dependent
    container.  Thus, this computation is sometimes called <quote>lazy
    evaluation</quote>.</para>
   </section>
***************
*** 558,589 ****
   <section id="concepts-computation_environment">
    <title>Computation Environment</title>
  
!   <para>A &pooma; program can execute on a wide variety of
!    computers.  The default <glossterm
     linkend="glossary-sequential"><firstterm>sequential computing
     environment</firstterm></glossterm> consists of one processor and
!    associated memory, as found on a personal computer.  In contrast,
!    a <glossterm
     linkend="glossary-distributed"><firstterm>distributed computing
     environment</firstterm></glossterm> may have multiple processors
     and multiple distributed or shared memories.  For example, some
     desktop computers have dual processors and shared memory, while a
!    large supercomputer may have thousands of processors, perhaps
!    with groups of eight sharing the same memory.</para>
  
    <para>Using distributed computation requires three things:
     <orderedlist>
      <listitem>
!      <para>the programmer must declare how container domains will
!       be distributed,</para>
      </listitem>
      <listitem>
       <para>&pooma; must be configured to use a communications
!       library, and </para>
       </listitem>
       <listitem>
! 		  <para>the &pooma; executable must be run using the
! 	library.</para>
      </listitem>
     </orderedlist>
     All of these were illustrated in <xref
--- 555,585 ----
   <section id="concepts-computation_environment">
    <title>Computation Environment</title>
  
!   <para>A &pooma; program can execute on a wide variety of computers.
!    The default <glossterm
     linkend="glossary-sequential"><firstterm>sequential computing
     environment</firstterm></glossterm> consists of one processor and
!    its associated memory, as found on a personal computer.  In
!    contrast, a <glossterm
     linkend="glossary-distributed"><firstterm>distributed computing
     environment</firstterm></glossterm> may have multiple processors
     and multiple distributed or shared memories.  For example, some
     desktop computers have dual processors and shared memory, while a
!    large supercomputer may have thousands of processors, perhaps with
!    groups of eight sharing the same memory.</para>
  
    <para>Using distributed computation requires three things:
     <orderedlist>
      <listitem>
!      <para>The program must declare how container domains will
!       be distributed.</para>
      </listitem>
      <listitem>
       <para>&pooma; must be configured to use a communications
!       library.</para>
       </listitem>
       <listitem>
!       <para>The &pooma; executable must be run using the library.</para>
      </listitem>
     </orderedlist>
     All of these were illustrated in <xref
***************
*** 592,604 ****
     linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
     illustrates the &pooma; distributed computation model.  <xref
     linkend="concepts-containers-distributed_declarations"></xref>
!    described how to declare containers with distributed domains.
!    Detailed instructions how to configure &pooma; for distributed
!    computation appear in <xref linkend="installation"></xref>.
!    Detailed instructions how to run distributed &pooma; executables
!    appear in <xref linkend="parallel"></xref>.  Here we present
!    three concepts for distributed computation: patches, context, and
!    a communication library.</para>
  
    <para>A partition divides a container's domain into disjoint
     <glossterm
--- 588,596 ----
     linkend="tutorial-array_distributed-doof2d-distributed_model"></xref>
     illustrates the &pooma; distributed computation model.  <xref
     linkend="concepts-containers-distributed_declarations"></xref>
!    described how to declare containers with distributed domains.  Here
!    we present three concepts for distributed computation: patches,
!    context, and a communication library.</para>
  
    <para>A partition divides a container's domain into disjoint
     <glossterm
***************
*** 611,618 ****
  
    <para>A <glossterm
     linkend="glossary-context"><firstterm>context</firstterm></glossterm>
!    is a collection of shared memory and processors that can execute
!    a program of a portion of a program.  It can have one or more
     processors, but all these processors must access the same shared
     memory.  Usually the computer and its operating system, not the
     programmer, determine the available contexts.</para>
--- 603,610 ----
  
    <para>A <glossterm
     linkend="glossary-context"><firstterm>context</firstterm></glossterm>
!    is a collection of shared memory and processors that can execute a
!    program or a portion of a program.  It can have one or more
     processors, but all these processors must access the same shared
     memory.  Usually the computer and its operating system, not the
     programmer, determine the available contexts.</para>
***************
*** 625,630 ****
     user.  &pooma; works with the Message Passing Interface (&mpi;)
     Communications Library (FIXME: xref linkend="mpi99", <ulink
     url="http://www-unix.mcs.anl.gov/mpi/"></ulink>) and the &mm;
!    Shared Memory Library.</para>
    </section>
  </chapter>
--- 617,623 ----
     user.  &pooma; works with the Message Passing Interface (&mpi;)
     Communications Library (FIXME: xref linkend="mpi99", <ulink
     url="http://www-unix.mcs.anl.gov/mpi/"></ulink>) and the &mm;
!    Shared Memory Library.  See <xref
!    linkend="installation-distributed_computing"></xref> for details.</para>
    </section>
  </chapter>
Index: glossary.xml
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/glossary.xml,v
retrieving revision 1.2
diff -c -p -r1.2 glossary.xml
*** glossary.xml	2001/12/13 04:04:05	1.2
--- glossary.xml	2001/12/14 04:12:54
***************
*** 341,347 ****
    <glossentry id="glossary-interval">
     <glossterm>interval</glossterm>
     <glossdef>
!     <para>a set of integral points between two values.  This domain
      is frequently represented using mathematical interval notation
      [a,b] even though it contains only the integral points, e.g., a,
      a+1, a+2, &hellip;, b.  It is also generalized to the tensor
--- 341,347 ----
    <glossentry id="glossary-interval">
     <glossterm>interval</glossterm>
     <glossdef>
!     <para>a set of integral points between two endpoints.  This domain
      is frequently represented using mathematical interval notation
      [a,b] even though it contains only the integral points, e.g., a,
      a+1, a+2, &hellip;, b.  It is also generalized to the tensor
Index: makefile
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/makefile,v
retrieving revision 1.2
diff -c -p -r1.2 makefile
*** makefile	2001/12/11 20:36:13	1.2
--- makefile	2001/12/14 04:12:54
*************** CXXFLAGS= -g -Wall -pedantic -W -Wstrict
*** 25,30 ****
--- 25,32 ----
  
  all: manual.ps
  
+ manual.dvi: manual.xml concepts.xml tutorial.xml
+ 
  %.all:	%.ps %.pdf %.html
  	chmod 644 $*.ps $*.pdf
  	mv $*.ps $*.pdf $*
Index: manual.xml
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/manual.xml,v
retrieving revision 1.2
diff -c -p -r1.2 manual.xml
*** manual.xml	2001/12/13 04:04:05	1.2
--- manual.xml	2001/12/14 04:12:57
***************
*** 356,365 ****
    <chapter id="sequential">
     <title>Writing Sequential Programs</title>
  
!    <para>QUESTIONS: How do I arrange this section?  What material do I
!    include?  What other books or models can I follow?</para>
  
  <!-- HERE -->
  
     <para>&pooma; can reorder computations to permit more efficient
     computation.  When running a sequential program, reordering may
--- 356,529 ----
    <chapter id="sequential">
     <title>Writing Sequential Programs</title>
  
!    <para>Proposed order.  Basically follow the order in the proposed
!    reference section.
!     <orderedlist>
!      <listitem><para>starting, stopping</para></listitem>
!      <listitem><para>&array;</para></listitem>
!      <listitem><para>&dynamicarray;</para></listitem>
!      <listitem><para>&field;</para></listitem>
!      <listitem><para>&vector;</para></listitem>
!      <listitem><para>&matrix;</para></listitem>
!      <listitem><para>&tensor;</para></listitem>
!      <listitem><para>engine</para></listitem>
!      <listitem><para>domain</para></listitem>
!      <listitem><para>correctness, e.g., <function>PAssert</function></para></listitem>
!      <listitem><para>&pooma; command-line options</para></listitem>
!     </orderedlist>
!     Include views of containers in the appropriate sections.</para>
  
  <!-- HERE -->
+ 
+    <para><emphasis>&c;: A Reference Manual</emphasis> uses this
+    structure for &c; libraries:
+     <orderedlist>
+      <listitem>
+       <para>function declarations, separated by rules from rest of text</para>
+      </listitem>
+      <listitem>
+       <para>text explanation</para>
+      </listitem>
+      <listitem>
+       <para>table of structure members if appropriate</para>
+      </listitem>
+      <listitem>
+       <para>example</para>
+      </listitem>
+     </orderedlist>
+    </para>
+ 
+    <para><emphasis>STL Tutorial and Reference Guide</emphasis>, second
+    edition, uses this structure for STL functions:
+     <orderedlist>
+      <listitem>
+       <para>text description with declaration mixed in</para>
+      </listitem>
+      <listitem>
+       <para>example program mixed into text.  It is an entire program,
+       not a program fragment.</para>
+      </listitem>
+     </orderedlist>
+    </para>
+ 
+    <para>A tutorial chapter for containers has
+     <orderedlist>
+      <listitem>
+       <para>explanation of template types</para>
+      </listitem>
+      <listitem>
+       <para>bulleted list of container types</para>
+      </listitem>
+      <listitem>
+       <para>example constructors</para>
+      </listitem>
+      <listitem>
+       <para>example programs</para>
+      </listitem>
+      <listitem>
+       <para>member and related functions with example programs</para>
+      </listitem>
+      <listitem>
+       <para>list of accessors and relation functions</para>
+      </listitem>
+     </orderedlist>
+    </para>
+ 
+    <para>The reference chapter for containers has
+     <orderedlist>
+      <listitem>
+       <para>a section listing common members and types for all containers</para>
+      </listitem>
+      <listitem>
+       <para>a section listing common member functions for all containers</para>
+      </listitem>
+      <listitem>
+       <para>requirements for various container specialties</para>
+      </listitem>
+     </orderedlist>
+     The section describing <type>vector</type>s contains
+     <orderedlist>
+      <listitem>
+       <para>files (header files)</para>
+      </listitem>
+      <listitem>
+       <para>class declaration</para>
+      </listitem>
+      <listitem>
+       <para>description</para>
+      </listitem>
+      <listitem>
+       <para>type definitions</para>
+      </listitem>
+      <listitem>
+       <para>constructors, destructors, and related functions</para>
+      </listitem>
+      <listitem>
+       <para>comparison operators</para>
+      </listitem>
+      <listitem>
+       <para>element access member functions</para>
+      </listitem>
+      <listitem>
+       <para>insert and erase member functions</para>
+      </listitem>
+      <listitem>
+       <para>notes</para>
+      </listitem>
+     </orderedlist>
+    </para>
+ 
+    <para>Josuttis's <emphasis>The &cc; Standard Library: A Tutorial
+    and Reference</emphasis> uses this structure for its STL container
+    chapter:
+     <orderedlist>
+      <listitem>
+       <para>short introduction</para>
+      </listitem>
+      <listitem>
+       <para>common container abilities</para>
+      </listitem>
+      <listitem>
+       <para>common container operations (with table)</para>
+      </listitem>
+      <listitem>
+       <para>vector abilities</para>
+      </listitem>
+      <listitem>
+       <para>vector operations:
+        <orderedlist>
+ 	<listitem>
+ 	 <para>create, copy, and destroy operations (mostly table)</para>
+ 	</listitem>
+ 	<listitem>
+ 	 <para>nonmodifying operations (mostly table)</para>
+ 	</listitem>
+ 	<listitem>
+ 	 <para>assignments (mostly table)</para>
+ 	</listitem>
+ 	<listitem>
+ 	 <para>element access (mostly table)</para>
+ 	</listitem>
+ 	<listitem>
+ 	 <para>iterator functions (mostly table)</para>
+ 	</listitem>
+ 	<listitem>
+ 	 <para>inserting and removing elements (mostly table)</para>
+ 	</listitem>
+        </orderedlist>
+       </para>
+      </listitem>
+      <listitem>
+       <para>using vectors as ordinary arrays</para>
+      </listitem>
+      <listitem>
+       <para>exception handling</para>
+      </listitem>
+      <listitem>
+       <para>example program</para>
+      </listitem>
+     </orderedlist>
+    </para>
  
     <para>&pooma; can reorder computations to permit more efficient
     computation.  When running a sequential program, reordering may
Index: tutorial.xml
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/tutorial.xml,v
retrieving revision 1.1
diff -c -p -r1.1 tutorial.xml
*** tutorial.xml	2001/12/11 20:36:13	1.1
--- tutorial.xml	2001/12/14 04:12:58
***************
*** 518,524 ****
   <section id="tutorial-array_stencil">
    <title>Stencil &array; Implementation</title>
  
!   <para>Many computations are local, computing a &array;'s value by
    using close-by &array; values.  Encapsulating this computation in
    a stencil can yield faster code because the compiler can determine
    all accesses come from the same array.  Each stencil consists of a
--- 518,524 ----
   <section id="tutorial-array_stencil">
    <title>Stencil &array; Implementation</title>
  
!   <para>Many computations are local, computing an &array;'s value by
    using close-by &array; values.  Encapsulating this computation in
    a stencil can yield faster code because the compiler can determine
    all accesses come from the same array.  Each stencil consists of a
Index: figures/concepts.mp
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/figures/concepts.mp,v
retrieving revision 1.2
diff -c -p -r1.2 concepts.mp
*** figures/concepts.mp	2001/12/13 04:04:05	1.2
--- figures/concepts.mp	2001/12/14 04:12:58
*************** endfig;
*** 116,122 ****
  
  %% Comparisons Between Mathematical Concept And Computational Implementation of Arrays and Fields
  beginfig(101)
!   numeric unit; unit = 0.9cm;
    numeric vertSpace; vertSpace = 2.6unit;   % vertical space between sections
    numeric horizSpace; horizSpace = 8unit; % horizontal space between sections
    
--- 116,122 ----
  
  %% Comparisons Between Mathematical Concept And Computational Implementation of Arrays and Fields
  beginfig(101)
!   numeric unit; unit = 0.8cm;
    numeric vertSpace; vertSpace = 2.6unit;   % vertical space between sections
    numeric horizSpace; horizSpace = 8unit; % horizontal space between sections
    
*************** beginfig(101)
*** 137,144 ****
    endfor
  
    % Create and layout text boxes.
!   boxit.l1(btex \strut mathematical concept etex);
!   boxit.l2(btex \strut computational implementation etex);
    boxit.l3(btex \strut \type{Array}: etex);
    boxit.l4(btex \strut $\mbox{index} \mapsto \mbox{value}$ etex);
    boxit.l6(btex \strut \type{Field}: etex);
--- 137,144 ----
    endfor
  
    % Create and layout text boxes.
!   boxit.l1(btex \strut \underline{mathematical concept} etex);
!   boxit.l2(btex \strut \underline{computational implementation} etex);
    boxit.l3(btex \strut \type{Array}: etex);
    boxit.l4(btex \strut $\mbox{index} \mapsto \mbox{value}$ etex);
    boxit.l6(btex \strut \type{Field}: etex);
*************** beginfig(101)
*** 152,158 ****
    l1.w - l3.w = l4.w - l7.w = (0,vertSpace);
    l4.w - l3.e = l7.nw - l6.ne = (0,0);
    for t = 0 upto 1:
!     xpart(ia[t].w - l[3+3t].e) = 0.65horizSpace;
      ypart(ia[t].w - l[3+3t].c) = 0;
    endfor
    xpart(l10.w - l7.w) = 0;
--- 152,158 ----
    l1.w - l3.w = l4.w - l7.w = (0,vertSpace);
    l4.w - l3.e = l7.nw - l6.ne = (0,0);
    for t = 0 upto 1:
!     xpart(ia[t].w - l[3+3t].e) = 0.6horizSpace;
      ypart(ia[t].w - l[3+3t].c) = 0;
    endfor
    xpart(l10.w - l7.w) = 0;

From oldham at codesourcery.com  Mon Dec 17 18:15:52 2001
From: oldham at codesourcery.com (Jeffrey Oldham)
Date: Mon, 17 Dec 2001 10:15:52 -0800
Subject: Patch: More Typo Fixes
Message-ID: <20011217101552.B505@codesourcery.com>

More typo fixes in comments.

2001-Dec-17  Jeffrey D. Oldham  <oldham at codesourcery.com>

        * README: Fixed typos in 2.1.0 entry.
        * src/Evaluator/PatchKernel.h: Fix typo in overview comment.
        * src/Pooma/Pooma.h: Remove extraneous semicolon from comment.
         (initialize): Fix spelling mistake in comment.
        * src/Utilities/Options.h: Fix spelling mistake in "Utility
          functions" comment.

Not tested:     since only comments were changed
Applied to      mainline
Approved by     Jim Crotinger and Mark Mitchell

Thanks,
Jeffrey D. Oldham
oldham at codesourcery.com
-------------- next part --------------
Index: README
===================================================================
RCS file: /home/pooma/Repository/r2/README,v
retrieving revision 1.60
diff -c -p -r1.60 README
*** README	2001/10/05 01:29:02	1.60
--- README	2001/12/17 16:44:26
*************** Pooma includes two models for a Geometry
*** 472,478 ****
    o NoGeometry<Dim>
      - only includes positions at which the field is defined
  
! DiscreteGoemetry depends on a Centering concept. Pooma II meshes (see
  below) do not know about centering - it enters at the geometry
  level. Centering is simply a mechanism to determine where the points
  of the Field are defined, relative to the mesh points. Pooma 2.1.0
--- 472,478 ----
    o NoGeometry<Dim>
      - only includes positions at which the field is defined
  
! DiscreteGeometry depends on a Centering concept. Pooma II meshes (see
  below) do not know about centering - it enters at the geometry
  level. Centering is simply a mechanism to determine where the points
  of the Field are defined, relative to the mesh points. Pooma 2.1.0
*************** POOMA 2.1.0 implements the following int
*** 699,705 ****
  CHANGES TO TENSOR CLASS AND NEW TINYMATRIX CLASS
  ------------------------------------------------
  
! The Tensor class now takes only one parameter to specify it's size; it 
  represents a square (D x D) mathematical tensor. The class declaration 
  is
  
--- 699,705 ----
  CHANGES TO TENSOR CLASS AND NEW TINYMATRIX CLASS
  ------------------------------------------------
  
! The Tensor class now takes only one parameter to specify its size; it 
  represents a square (D x D) mathematical tensor. The class declaration 
  is
  
Index: src/Evaluator/PatchKernel.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Evaluator/PatchKernel.h,v
retrieving revision 1.18
diff -c -p -r1.18 PatchKernel.h
*** src/Evaluator/PatchKernel.h	2000/06/08 22:16:13	1.18
--- src/Evaluator/PatchKernel.h	2001/12/17 16:44:26
***************
*** 38,44 ****
  
  //-----------------------------------------------------------------------------
  // Overview: 
! // A PatchKernel encapsulates perfoming operations on a patch of an expression.
  //-----------------------------------------------------------------------------
  
  //-----------------------------------------------------------------------------
--- 38,45 ----
  
  //-----------------------------------------------------------------------------
  // Overview: 
! // A PatchKernel encapsulates performing operations on a patch of an
! // expression.
  //-----------------------------------------------------------------------------
  
  //-----------------------------------------------------------------------------
Index: src/Pooma/Pooma.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Pooma/Pooma.h,v
retrieving revision 1.31
diff -c -p -r1.31 Pooma.h
*** src/Pooma/Pooma.h	2001/11/05 23:46:29	1.31
--- src/Pooma/Pooma.h	2001/12/17 16:44:26
***************
*** 47,53 ****
  //
  //   Pooma::printStats
  //   Pooma::debugLevel
! //   Pooma::infoMessages;
  //   Pooma::warnMessages
  //   Pooma::errorMessages
  //   Pooma::logMessages
--- 47,53 ----
  //
  //   Pooma::printStats
  //   Pooma::debugLevel
! //   Pooma::infoMessages
  //   Pooma::warnMessages
  //   Pooma::errorMessages
  //   Pooma::logMessages
*************** namespace Pooma {
*** 294,300 ****
  
    // Initialize POOMA, using the given Options container instead of argc,argv.
    // If the 2nd argument is true, also initialize the run-time system. 
!   // If the 3rd argument is true, call arch-specific initalize().
    // Return success. 
  
    bool initialize(Pooma::Options &opts, bool initRTS = true, 
--- 294,300 ----
  
    // Initialize POOMA, using the given Options container instead of argc,argv.
    // If the 2nd argument is true, also initialize the run-time system. 
!   // If the 3rd argument is true, call arch-specific initialize().
    // Return success. 
  
    bool initialize(Pooma::Options &opts, bool initRTS = true, 
Index: src/Utilities/Options.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Utilities/Options.h,v
retrieving revision 1.4
diff -c -p -r1.4 Options.h
*** src/Utilities/Options.h	2000/06/30 02:00:16	1.4
--- src/Utilities/Options.h	2001/12/17 16:44:26
*************** private:
*** 285,291 ****
  // Utility functions.
  //============================================================
  
! // These used to be private methodes in the Options class, but they 
  // are generally useful for parsing options, so they're now in the Pooma
  // namespace.
  
--- 285,291 ----
  // Utility functions.
  //============================================================
  
! // These used to be private methods in the Options class, but they 
  // are generally useful for parsing options, so they're now in the Pooma
  // namespace.
  

From oldham at codesourcery.com  Mon Dec 17 18:28:52 2001
From: oldham at codesourcery.com (Jeffrey Oldham)
Date: Mon, 17 Dec 2001 10:28:52 -0800
Subject: Manual Patch: New Introductory Chapter
Message-ID: <20011217102852.C505@codesourcery.com>

This patch mainly adds an introductory chapter and a very small part of
the sequential program chapter.

2001-Dec-17  Jeffrey D. Oldham  <oldham at codesourcery.com>

	* concepts.xml: Minor wordsmithing fixes, e.g., removal of old
	temporary paragraphs, spelling changes, and better use of entities.
	* glossary.xml: s/multi-processor/multiprocessor/
	(architecture): New entry.
	(first class): Refill.
	* introduction.xml: New introductory chapter.
	* makefile (manual.dvi): Add dependence on introduction.xml and
	glossary.xml.
	* manual.xml: Add a few new entity declarations and use them.
	Move introductory chapter material to introduction.xml.  Begin
	writing sequential program chapter.  Add a few bibliographic
	entries.
	* tutorial.xml: Add more uses of entities changed in manual.xml.
	* figures/introduction.mp: New figure illustrating role of Pooma
	in science/math process.

Applied to	mainline
Approved by	me!

Thanks,
Jeffrey D. Oldham
oldham at codesourcery.com
-------------- next part --------------
Index: concepts.xml
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/concepts.xml,v
retrieving revision 1.2
diff -c -p -r1.2 concepts.xml
*** concepts.xml	2001/12/14 04:18:13	1.2
--- concepts.xml	2001/12/17 16:56:50
***************
*** 1,9 ****
  <chapter id="concepts">
   <title>Overview of &pooma; Concepts</title>
  
-  <para>FIXME: How does multi-threaded computation fit into the
-  model?</para>
- 
   <para>In the previous chapter, we presented several different
   implementations of the &doof2d; simulation program.  The
   implementations illustrate the various containers, computation modes,
--- 1,6 ----
***************
*** 33,39 ****
      <term>computation environment</term>
      <listitem>
       <para>description of resources for computing, e.g., single
!      processor or multi-processor.</para>
      </listitem>
     </varlistentry>
    </variablelist>
--- 30,36 ----
      <term>computation environment</term>
      <listitem>
       <para>description of resources for computing, e.g., single
!      processor or multiprocessor.</para>
      </listitem>
     </varlistentry>
    </variablelist>
***************
*** 115,121 ****
    containers.</para>
  
    <para>This section describes many concepts, but one need not
!   understand them all to begin programming with the &pooma; Toolkit.
    First, we introduce the different &pooma;'s containers and describe
    how to choose an appropriate one for a particular task.  <xref
    linkend="concepts-sequential_containers-declarations-dependences"></xref>
--- 112,118 ----
    containers.</para>
  
    <para>This section describes many concepts, but one need not
!   understand them all to begin programming with the &poomatoolkit;.
    First, we introduce the different &pooma;'s containers and describe
    how to choose an appropriate one for a particular task.  <xref
    linkend="concepts-sequential_containers-declarations-dependences"></xref>
***************
*** 361,367 ****
     memory.  The layout specifies the processors and memory to use for
     each particular index.  A container's layout for a uniprocessor
     implementation consists of its domain, the processor, and its
!    memory.  For a multi-processor implementation, the layout maps
     portions of the domain to (possibly different) processors and
     memory.</para>
  
--- 358,364 ----
     memory.  The layout specifies the processors and memory to use for
     each particular index.  A container's layout for a uniprocessor
     implementation consists of its domain, the processor, and its
!    memory.  For a multiprocessor implementation, the layout maps
     portions of the domain to (possibly different) processors and
     memory.</para>
  
***************
*** 422,428 ****
  
     <para>In the previous section, we introduced the concepts important
     when declaring containers for use on uniprocessor computers.  When
!    using multi-processor computers, we augment these concepts with
     those for distributed computation.  Reading this section is
     important only for running a program on multiple processors.  Many
     of these concepts were introduced in <xref
--- 419,425 ----
  
     <para>In the previous section, we introduced the concepts important
     when declaring containers for use on uniprocessor computers.  When
!    using multiprocessor computers, we augment these concepts with
     those for distributed computation.  Reading this section is
     important only for running a program on multiple processors.  Many
     of these concepts were introduced in <xref
***************
*** 434,447 ****
     distributed container.</para>
  
     <para>As we noted in <xref
!    linkend="tutorial-array_distributed"></xref>, a &pooma;
!    programmer must specify how each container's domain should be
!    distributed among the available processors and memory spaces.
!    Using this information, the Toolkit automatically distributes the
!    data among the available processors and handles any required
!    communication among them.  The three concepts necessary for
!    declaring distributed containers are a partition, a guard layer,
!    and a context mapper tag.</para>
  
     <para>A <glossterm
     linkend="glossary-partition"><firstterm>partition</firstterm></glossterm>
--- 431,444 ----
     distributed container.</para>
  
     <para>As we noted in <xref
!    linkend="tutorial-array_distributed"></xref>, a &pooma; programmer
!    must specify how each container's domain should be distributed
!    among the available processors and memory spaces.  Using this
!    information, the &toolkit; automatically distributes the data among
!    the available processors and handles any required communication
!    among them.  The three concepts necessary for declaring distributed
!    containers are a partition, a guard layer, and a context mapper
!    tag.</para>
  
     <para>A <glossterm
     linkend="glossary-partition"><firstterm>partition</firstterm></glossterm>
***************
*** 615,622 ****
     &pooma; uses the communication library to copy information among
     contexts, all of which is hidden from both the programmer and the
     user.  &pooma; works with the Message Passing Interface (&mpi;)
!    Communications Library (FIXME: xref linkend="mpi99", <ulink
!    url="http://www-unix.mcs.anl.gov/mpi/"></ulink>) and the &mm;
     Shared Memory Library.  See <xref
     linkend="installation-distributed_computing"></xref> for details.</para>
    </section>
--- 612,620 ----
     &pooma; uses the communication library to copy information among
     contexts, all of which is hidden from both the programmer and the
     user.  &pooma; works with the Message Passing Interface (&mpi;)
!    Communications Library 
! <!-- FIXME: xref linkend="mpi99" -->
!    (<ulink url="http://www-unix.mcs.anl.gov/mpi/"></ulink>) and the &mm;
     Shared Memory Library.  See <xref
     linkend="installation-distributed_computing"></xref> for details.</para>
    </section>
Index: glossary.xml
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/glossary.xml,v
retrieving revision 1.3
diff -c -p -r1.3 glossary.xml
*** glossary.xml	2001/12/14 04:18:13	1.3
--- glossary.xml	2001/12/17 16:56:50
***************
*** 17,22 ****
--- 17,31 ----
  
   <glossdiv id="glossary-a">
    <title>A</title>
+   <glossentry id="glossary-architecture">
+    <glossterm>architecture</glossterm>
+    <glossdef>
+     <para>particular hardware (processor) interface.  Examples
+     architectures include <quote>linux</quote>, <quote>sgin32</quote>,
+     <quote>sgi64</quote>, and <quote>sun</quote>.</para>
+    </glossdef>
+   </glossentry>
+ 
    <glossentry id="glossary-array">
     <glossterm>&array;</glossterm>
     <glossdef>
***************
*** 25,32 ****
      ignoring the time to compute the values if applicable.  &array;s
      are <link linkend="glossary-first_class">first-class
      object</link>s.  <link
! 			    linkend="glossary-dynamicarray">&dynamicarray;</link>s and <link
! 											     linkend="glossary-field">&field;</link>s generalize &array;.</para>
      <glossseealso otherterm="glossary-dynamicarray">&dynamicarray;</glossseealso>
      <glossseealso otherterm="glossary-field">&field;</glossseealso>
     </glossdef>
--- 34,42 ----
      ignoring the time to compute the values if applicable.  &array;s
      are <link linkend="glossary-first_class">first-class
      object</link>s.  <link
!     linkend="glossary-dynamicarray">&dynamicarray;</link>s and <link
!     linkend="glossary-field">&field;</link>s generalize
!     &array;.</para>
      <glossseealso otherterm="glossary-dynamicarray">&dynamicarray;</glossseealso>
      <glossseealso otherterm="glossary-field">&field;</glossseealso>
     </glossdef>
***************
*** 165,171 ****
     <glossdef>
      <para>computing environment with one or more processors each
      having associated memory, possibly shared.  In some contexts, it
!     refers to strictly multi-processor computation.</para>
      <glossseealso otherterm="glossary-computing_environment">computing environment</glossseealso>
      <glossseealso otherterm="glossary-sequential">sequential computing environment</glossseealso>
     </glossdef>
--- 175,181 ----
     <glossdef>
      <para>computing environment with one or more processors each
      having associated memory, possibly shared.  In some contexts, it
!     refers to strictly multiprocessor computation.</para>
      <glossseealso otherterm="glossary-computing_environment">computing environment</glossseealso>
      <glossseealso otherterm="glossary-sequential">sequential computing environment</glossseealso>
     </glossdef>
***************
*** 364,370 ****
      <para>a map from an index to processor(s) and memory used to
      compute the container's associated value.  For a uniprocessor
      implementation, a container's layout always consists of its
!     domain, the processor, and its memory.  For a multi-processor
      implementation, the layout maps portions of the domain to
      (possibly different) processors and memory.</para>
      <glossseealso otherterm="glossary-container">container</glossseealso>
--- 374,380 ----
      <para>a map from an index to processor(s) and memory used to
      compute the container's associated value.  For a uniprocessor
      implementation, a container's layout always consists of its
!     domain, the processor, and its memory.  For a multiprocessor
      implementation, the layout maps portions of the domain to
      (possibly different) processors and memory.</para>
      <glossseealso otherterm="glossary-container">container</glossseealso>
***************
*** 572,580 ****
      <para>a container derived from another.  The former's domain is a
      subset of the latter's, but, where the domains intersect,
      accessing a value through the view is the same as accessing it
!     through the original container.  Only &array;s, &dynamicarray;s,
!     and &field;s support views.</para>
!     <glossseealso otherterm="glossary-container">container</glossseealso>
     </glossdef>
    </glossentry>
   </glossdiv>
--- 582,591 ----
      <para>a container derived from another.  The former's domain is a
      subset of the latter's, but, where the domains intersect,
      accessing a value through the view is the same as accessing it
!     through the original container.  In Fortran&nbsp;90, these are
!     called array sections.  Only &array;s, &dynamicarray;s, and
!     &field;s support views.</para> <glossseealso
!     otherterm="glossary-container">container</glossseealso>
     </glossdef>
    </glossentry>
   </glossdiv>
Index: introduction.xml
===================================================================
RCS file: introduction.xml
diff -N introduction.xml
*** /dev/null	Fri Mar 23 21:37:44 2001
--- introduction.xml	Mon Dec 17 09:56:51 2001
***************
*** 0 ****
--- 1,348 ----
+ <chapter id="introduction">
+  <title>Introduction</title>
+ 
+  <para>The Parallel Object-Oriented Methods and Applications
+  <acronym>POOMA</acronym> &toolkitcap; is a &cc; &toolkit; for
+  writing high-performance scientific programs for sequential and
+  distributed computation.  The &toolkit; provides a variety of
+  tools:
+  <itemizedlist spacing="compact">
+    <listitem>
+     <para>containers and other abstractions suitable for scientific
+     computation,</para>
+    </listitem>
+    <listitem>
+     <para>several container storage classes to reduce a program's
+     storage requirements,</para>
+    </listitem>
+    <listitem>
+     <para>support for a variety of computation modes including
+     data-parallel expressions, stencil-based computations, and lazy
+     evaluation,</para>
+    </listitem>
+    <listitem>
+     <para>support for writing parallel and distributed programs,</para>
+    </listitem>
+    <listitem>
+     <para>automatic creation of all interprocessor communication for
+     parallel and distributed programs, and</para>
+    </listitem>
+    <listitem>
+     <para>automatic out-of-order execution and loop rearrangement
+     for fast program execution.</para>
+    </listitem>
+   </itemizedlist>
+  Since the &toolkit; provides high-level abstractions, &pooma;
+  programs are much shorter than corresponding &fortran; or &c;
+  programs, requiring less time to write and less time to debug.
+  Using these high-level abstractions, the same code runs on a wide
+  variety of computers almost as fast as carefully crafted
+  machine-specific hand-written programs.  The &toolkit; is freely
+  available, open-source software compatible with any modern &cc;
+  compiler.</para>
+ 
+  <formalpara><title>&pooma; Goals.</title>
+   <para>The goals for the &poomatoolkit; have remained unchanged
+   since its inception in 1994:
+   <orderedlist>
+    <listitem>
+     <para>Code portability across serial, distributed, and parallel
+     architectures with no change to source code.</para>
+    </listitem>
+    <listitem>
+     <para>Development of reusable, cross-problem-domain components
+     to enable rapid application development.</para>
+    </listitem>
+    <listitem>
+     <para>Code efficiency for kernels and components relevant to
+     scientific simulation.</para>
+    </listitem>
+    <listitem>
+     <para>[&toolkitcap;] design and development driven by
+     applications from a diverse set of scientific problem
+     domains.</para>
+    </listitem>
+    <listitem>
+     <para>Shorter time from problem inception to working parallel
+     simulations.</para>
+ <!-- FIXME: Add citation to pooma95, p.&nbsp;3 -->
+    </listitem>
+   </orderedlist>
+  </para>
+  </formalpara>
+ 
+  <formalpara><title>Code Portability for Sequential and Distributed Programs.</title>
+  <para>&pooma; programs run on sequential, distributed, and parallel
+  computers with no change in source code.  The programmer writes two
+  or three lines specifying how each container's domain should be
+  distributed among available processors.  Using these directives and
+  run-time information about the computer's configuration, the
+  &toolkit; automatically distributes pieces of the container
+  domains, called <firstterm>patch</firstterm>es, among the available
+  processors.  If a computation needs values from another patch,
+  &pooma; automatically passes the value to the place it is needed.
+  The same program, and even the same executable, works regardless of
+  the number of the available processors and the size of the
+  containers' domains.  A programmer interested in only sequential
+  execution can omit the two or three lines specifying how the
+  domains are to be distributed.</para>
+  </formalpara>
+ 
+  <figure float="1" id="introduction-science_algorithms">
+   <title>Science, Algorithms, Engineering, and &pooma;</title>
+   <mediaobject>
+    <imageobject>
+     <imagedata fileref="figures/introduction.101" format="EPS" align="center"></imagedata>
+    </imageobject>
+    <textobject>
+     <phrase>how &pooma; helps translate algorithms into programs</phrase>
+    </textobject>
+    <caption>
+     <para>In the translation from theoretical science and math to
+     computational science and math to computer programs, &pooma;
+     containers eases the translation of algorithms to computer
+     programs.</para>
+    </caption>
+   </mediaobject>
+  </figure>
+ 
+  <formalpara><title>Rapid Application Development.</title>
+  <para>The &poomatoolkit; is designed to enable rapid development of
+  scientific and distributed applications.  For example, its vector,
+  matrix, and tensor classes model the corresponding mathematical
+  concepts.  Its &array; and &field; classes model the discrete
+  spaces and mathematical arrays frequently found in computational
+  science and math.  See <xref
+  linkend="introduction-science_algorithms"></xref>.  The left column
+  illustrates theoretical science and math, the middle column
+  computational science and math, and the right column computer
+  science implementations.  For example, theoretical physics
+  frequently uses continuous fields in three-dimension space, while
+  algorithms for the corresponding computational physics problem
+  usually uses discrete fields.  &pooma; containers, classes, and
+  functions ease the engineering to map these algorithms to computer
+  programs.  For example, the &pooma; &field; container models
+  discrete fields; both map locations in discrete space to values and
+  permit computations of spatial distances and values.  The &pooma;
+  &array; container models the mathematical concept of an array, used
+  in numerical analysis.</para>
+  </formalpara>
+ 
+  <para>&pooma; containers support a variety of computation modes,
+  easing transition of algorithms into code.  For example, many
+  algorithms for solving partial differential equations use
+  stencil-based computations.  &pooma; supports stencil-based
+  computations on &array;s and &field;s.  It also supports
+  data-parallel computation.  For computations where one &field;'s
+  values is a function of several other &field;'s values, the
+  programmer can specify a relation.  Relations are lazily evaluated;
+  whenever the dependent &field;'s values are needed and it is
+  related to a &field; whose values have changed, the former
+  &field;'s values are computed.  Lazy evaluation also assists
+  correctness by eliminating the (frequently forgotten) need for a
+  programmer to ensure a &field;'s values are up-to-date before being
+  used.</para>
+ 
+  <formalpara><title>Efficient Code.</title>
+  <para>&pooma; incorporates a variety of techniques to ensure it
+  produces code that executes as quickly as special-case,
+  hand-written code.
+ <!-- FIXME: Do I present execution numbers here? -->
+  These techniques include extensive use of templates, out-of-order
+  evaluation to permit communication and computation to overlap,
+  availability of guard layers to reduce processors' synchronicity,
+  and use of &pete; to produce fast inner loops.</para>
+  </formalpara>
+ 
+  <para>Using templates permits the expressiveness of using pointers
+  and function arguments but ensures as much as work as possible
+  occurs at compile time, not run time.  Also, more code is exposed
+  to the compiler's optimizer, further speeding execution.  For
+  example, use of template parameters to define the &pooma; &array;
+  container permits the use of specialized data storage classes
+  called engines, fast creation of views of a portion of an &array;,
+  and polymorphic indexing.  An &array;'s engine template parameter
+  specifies how data is stored and indexed.  Some &array;s expect
+  almost all values to be used, while others might be mostly empty.
+  In the latter case, using a specialized engine storing the few
+  nonzero values would greatly reduce space requirements.  Using
+  engines also permits fast creation of container views, known as
+  <firstterm>array sections</firstterm> in Fortran&nbsp;90.  A view's
+  engine is the same as the original container's engine, while the
+  view object maps its restricted domain to the original domain.
+  Space requirements and execution time are minimal.  Using templates
+  also permits containers to support polymorphic indexing, e.g.,
+  indexing both by integers and by three-dimensional coordinates.
+  For example, a container defers returning values to its engine
+  using a templatized index operator.  The engine can define indexing
+  functions with different function arguments, without the need to
+  add corresponding container functions.  Some of these features can
+  be expressed without using templates, but doing so increases
+  execution time.  For example, a container could have a pointer to
+  an engine object, but this requires a pointer dereference for each
+  operation.  Implementing polymorphic indexing without templates
+  would require adding virtual function corresponding to each of the
+  indexing functions.</para>
+ 
+ <!-- FIXME: Are the claims concerning out-of-order evaluation I make true? -->
+ 
+  <para>To ensure multiprocessor &pooma; programs execute quickly, it
+  is important that interprocessor communication overlaps with
+  intraprocessor computation as much as possible and communication is
+  minimized.  Asynchronous communication, out-of-order evaluation, and
+  use of guard layers all help achieve this.  &pooma; uses the
+  asynchronous communication facilities of the &cheetah; communication
+  library.  When a processor needs data stored or computed by another
+  processor, a message is sent between the two.  For synchronous
+  communication, the sender must issue an explicit send, and the
+  recipient must issue an explicit receive.  This synchronizes them.
+  &cheetah; permits the sender to put and get data without the
+  intervention of the remote site and also invoke functions at the
+  remote site to ensure the data is up-to-date.  Thus, out-of-order
+  evaluation must be supported.  Out-of-order evaluation has another
+  benefit: only computations directly or indirectly related to values
+  that are printed need occur.</para>
+ 
+  <para>Using guard layers also helps overlap communication and
+  computation.  For distributed computation, each container's domain is
+  split into pieces distributed among the available processors.
+  Frequently, computing a container value is local, involving just the
+  value itself and a few neighbors.  Computing a value near the edge of
+  a processor's domain may require knowing a few values from a
+  neighboring domain.  Guard layers permit these values to be copied
+  locally so they need not be repeatedly communicated.</para>
+ 
+  <para>&pooma; uses &pete; technology to ensure inner loops using
+  &pooma;'s object-oriented containers run as quickly as hand-coded
+  <!-- FIXME: Add a citation to Dr. Dobb's Journal article
+  pete-99. --> loops.  &pete; (the Portable Expression Template
+  Engine) uses expression-template technology to convert
+  data-parallel statements frequently found in the inner loops of
+  programs into efficient loops without any intermediate
+  computations.  For example, consider evaluating the <statement>A +=
+  -B + 2 * C;</statement> statement where <varname>A</varname> and
+  <varname>C</varname> are <type>vector&lt;double&gt;</type>s and
+  <varname>B</varname> is a <type>vector&lt;int&gt;</type>s.
+  Ordinary evaluation might introduce intermediaries for
+  <statement>-B</statement>, <statement>2*C</statement>, and their
+  sum.  The presence of these intermediaries in inner loops can
+  measurably slow evaluation.  To produce a loop without
+  intermediaries, &pete; stores each expression as a parse tree.  The
+  resulting parse trees can be combined into a larger parse tree.
+  Using its templates, the parse tree is converted, at compile time,
+  to an outer loop with contents corresponding to evaluating each
+  component of the result.  Thus, no intermediate values are computed
+  or stored.  For example, the code corresponding to <statement>A +=
+  -B + 2 * C;</statement> is 
+  <programlisting>
+  vector&lt;double&gt;::iterator iterA = A.begin();
+  vector&lt;int&gt;::const_iterator iterB = B.begin();
+  vector&lt;double&gt;::const_iterator iterC = C.begin();
+  while (iterA != A.end()) {
+    *iterA += -*iterB + 2 * *iterC;
+    ++iterA; ++iterB; ++iterC;
+  }
+  </programlisting>
+  Furthermore, since the code is available at compile-, not run-, time,
+  it can be further optimized, e.g., moving any loop-invariant code out
+  of the loop.</para>
+ 
+  <formalpara><title>Used for Diverse Set of Scientific Problems.</title>
+  <para>&pooma; has been used to solve a wide variety of scientific
+  problems.  Most recently, physicists at Los Alamos National
+  Laboratory implemented an entire library of hydrodynamics codes as
+  part of the U.S. government's Science-based Stockpile Stewardship
+  (<acronym>SBSS</acronym>) program to simulate nuclear weapons.
+  Other applications include a matrix solver, an accelerator code
+  simulating the dynamics of high-intensity charged particle beams in
+  linear accelerators, and a Monte Carlo neutron transport
+  code.</para>
+  </formalpara>
+ 
+  <formalpara><title>Easy Implementation.</title>
+  <para>&pooma;'s tools greatly reduce the time to implement
+  applications.  As we noted above, &pooma;'s containers and
+  expression syntax model the computational models and algorithms
+  most frequently found in scientific programs.  Using these
+  high-level tools which are known to be correct reduce the time
+  needed to debug programs.  Programmers can write and test programs
+  using their one or two-processor personal computers.  With no
+  additional work, the same program runs on computers with hundreds
+  of processors; the code is exactly the same, and the &toolkit;
+  automatically handles distribution of the data, all data
+  communication, and all synchronization.  Using all these tools
+  greatly reduces programming time.  For example, a team of two
+  physicists and two support people at Los Alamos National Laboratory
+  implemented a suite of hydrodynamics kernels in six months.  Their
+  work replaced the previous suite of less-powerful kernels which had
+  taken sixteen people several years to implement and debug.  Despite
+  not previously implementing any of the kernels, they averaged one
+  new kernel every three days, including the time to read the
+  corresponding scientific papers!</para>
+  </formalpara>
+ 
+  <section id="introduction-pooma_history">
+   <title>History of &pooma;</title>
+ 
+   <para>The &poomatoolkit; developed at Los Alamos National
+   Laboratory to assist nuclear fusion and fission research.
+   In&nbsp;1994, the &toolkit; grew out of the Object-Oriented
+   Particle Simulation (OOPS) class library developed for
+   particle-in-cell simulations.  The goals of the Framework, as it
+   was called at the time, were driven by the Numerical Tokamak's
+   <quote>Parallel Platform Paradox</quote>:
+   <blockquote>
+    <para>The average time required to implement a moderate-sized
+    application on a parallel computer architecture is equivalent to
+    the half-life of the latest parallel supercomputer.</para>
+   </blockquote>
+   The framework's goal of being able to quickly write efficient
+   scientific code that could be run on a wide variety of platforms
+   remains unchanged today.  Development, driven mainly by the
+   Advanced Computing Laboratory at Los Alamos, proceeded rapidly.
+   A matrix solver application was written using the framework.
+ <!-- FIXME: Add citation to pooma-sc95. -->
+   Support for hydrodynamics, Monte Carlo simulations, and molecular
+   dynamics modeling soon followed.</para>
+ 
+   <para>By&nbsp;1998, &pooma; was part of the U.S. Department of
+   Energy's Accelerated Strategic Computing Initiative
+   (<acronym>ASCI</acronym>).  The Comprehensive Test Ban Treaty
+   forbid nuclear weapons testing so they were instead simulated.
+   <acronym>ASCI</acronym>'s goal was to radically advance the state
+   of the art in high-performance computing and numerical simulations
+   so the nuclear weapon simulations could use 100-teraflop
+   computers.  A linear accelerator code <application
+   class='software'>linac</application> and a Monte Carlo neutron
+   transport code <application class='software'>MC++</application>
+   were written.
+ <!-- FIXME: Add citation to pooma-siam98. -->
+   </para>
+ 
+   <para>&pooma;&nbsp;2 involved a new conceptual framework and a
+   complete rewriting of the source code to improve performance.  The
+ <!-- FIXME: Add a citation to iscope98.pdf. -->
+   &array; class was introduced with its use of engines, separating
+   container use from container storage.  An asynchronous scheduler
+   permitted out-of-order execution to improve cache coherency.
+   Incorporating the <application class="software">Portable
+   Expression Template Engine</application> (<acronym>PETE</acronym>)
+   permitted faster loop execution.  Soon, container views and
+   <type>ConstantFunction</type> and <type>IndexFunction</type>
+   engines were added.  Release&nbsp;2.1.0 included &field;s with
+   their spatial extent and &dynamicarray;s with the ability to
+   dynamically change its domain size.  Support for particles and
+   their interaction with &field;s was added.  The &pooma; messaging
+   implementation was revised in release&nbsp;2.3.0.  Use of the
+   &cheetah; Library separated &pooma; from the actual messaging
+   library used.  Support for applications running on clusters of
+   computers was added.  During the past two years, the &field;
+   abstraction and implementation was improved to increase its
+   flexibility, add support for multiple values and materials in the
+   same cell, and permit lazy evaluation.  Simultaneously, the
+   execution speed of the inner loops was greatly increased.  The
+   particle code has not yet been ported to the new &field;
+   abstraction.</para>
+  </section>
+ 
+ </chapter>
Index: makefile
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/makefile,v
retrieving revision 1.3
diff -c -p -r1.3 makefile
*** makefile	2001/12/14 04:18:13	1.3
--- makefile	2001/12/17 16:56:51
*************** CXXFLAGS= -g -Wall -pedantic -W -Wstrict
*** 25,31 ****
  
  all: manual.ps
  
! manual.dvi: manual.xml concepts.xml tutorial.xml
  
  %.all:	%.ps %.pdf %.html
  	chmod 644 $*.ps $*.pdf
--- 25,31 ----
  
  all: manual.ps
  
! manual.dvi: manual.xml introduction.xml tutorial.xml concepts.xml glossary.xml
  
  %.all:	%.ps %.pdf %.html
  	chmod 644 $*.ps $*.pdf
Index: manual.xml
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/manual.xml,v
retrieving revision 1.3
diff -c -p -r1.3 manual.xml
*** manual.xml	2001/12/14 04:18:13	1.3
--- manual.xml	2001/12/17 16:56:54
***************
*** 30,35 ****
--- 30,37 ----
    <!-- Produce a notation for a double dash.  Without this, TeX produces an en-hyphen. -->
  <!ENTITY doof2d "<command>Doof2d</command>" >
    <!-- Produce a notation for the Doof2d program.  -->
+ <!ENTITY fortran "<application class='software'>Fortran</application>">
+   <!-- Produce a notation for the Fortran programming language.  -->
  <!ENTITY make "<application class='software'>Make</application>">
    <!-- Produce a notation for the GNU Make program.  -->
  <!ENTITY mm "<application class='software'>MM</application>">
***************
*** 42,48 ****
    <!-- Produce a notation for the PETE library.  -->
  <!ENTITY pooma "<application class='software'>POOMA</application>">
    <!-- Produce a notation for Pooma software.  -->
! <!ENTITY poomaToolkit "<application class='software'>POOMA Toolkit</application>">
    <!-- Produce a notation for the Pooma toolkit.  -->
  <!ENTITY purify "<application class='software'>Purify</application>">
    <!-- Produce a notation for the Purify library.  -->
--- 44,50 ----
    <!-- Produce a notation for the PETE library.  -->
  <!ENTITY pooma "<application class='software'>POOMA</application>">
    <!-- Produce a notation for Pooma software.  -->
! <!ENTITY poomatoolkit "<application class='software'>POOMA &toolkitcap;</application>">
    <!-- Produce a notation for the Pooma toolkit.  -->
  <!ENTITY purify "<application class='software'>Purify</application>">
    <!-- Produce a notation for the Purify library.  -->
***************
*** 53,58 ****
--- 55,64 ----
    <!-- Produce a notation for the C++ Standard Template Library software package.  -->
  <!ENTITY tau "<application class='software'>Tau</application>">
    <!-- Produce a notation for the Tau software package.  -->
+ <!ENTITY toolkit "toolkit">
+   <!-- Produce a notation for the name of the Pooma software.  -->
+ <!ENTITY toolkitcap "Toolkit">
+   <!-- Produce a capitalized version of &toolkit;.  -->
  
  <!-- Type Entity Declarations -->
  
***************
*** 74,79 ****
--- 80,88 ----
    <!-- The "DynamicArray" type. -->
  <!ENTITY engine "<type>Engine</type>">
    <!-- The "Engine" type. -->
+ <!ENTITY false "<statement>false</statement>">
+   <!-- The false Boolean value. -->
+   <!-- Modify its tag to the appropriate one. -->
  <!ENTITY field "<type>Field</type>">
    <!-- The "Field" type. -->
  <!ENTITY inform "<type>Inform</type>">
***************
*** 88,99 ****
--- 97,113 ----
    <!-- The Pooma matrix type. -->
  <!ENTITY multipatch "<type>MultiPatch</type>">
    <!-- The "MultiPatch" engine without template parameters. -->
+ <!ENTITY options "<type>Options</type>">
+   <!-- The &pooma; options type. -->
  <!ENTITY replicatedtag "<type>ReplicatedTag</type>">
    <!-- The ReplicatedTag Layout type. -->
  <!ENTITY stencil "<type>Stencil</type>">
    <!-- The "Stencil" type. -->
  <!ENTITY tensor "<type>Tensor</type>">
    <!-- The Pooma tensor type. -->
+ <!ENTITY true "<statement>true</statement>">
+   <!-- The true Boolean value. -->
+   <!-- Modify its tag to the appropriate one. -->
  <!ENTITY vector "<type>Vector</type>">
    <!-- The "Vector" type. -->
  
***************
*** 135,171 ****
    <!-- spelling: dependence, not dependency -->
    <!-- spelling: element-wise, not elementwise -->
    <!-- phrase: function object, not functor -->
    <!-- spelling: multidimensional, not multi-dimensional -->
    <!-- spelling: multiprocessor, not multi-processor -->
    <!-- spelling: nonzero, not non-zero -->
  
  <!-- External File Entities -->
! <!ENTITY doof2d-c-element SYSTEM "./programs/Doof2d-C-element-annotated.cpp">
    <!-- hand-coded Doof2d implementation -->
! <!ENTITY doof2d-array-element SYSTEM "./programs/Doof2d-Array-element-annotated.cpp">
    <!-- Array element-wise Doof2d implementation -->
! <!ENTITY doof2d-array-parallel SYSTEM "./programs/Doof2d-Array-parallel-annotated.cpp">
    <!-- Array data-parallel Doof2d implementation -->
! <!ENTITY doof2d-array-stencil SYSTEM "./programs/Doof2d-Array-stencil-annotated.cpp">
    <!-- Array stencil Doof2d implementation -->
! <!ENTITY doof2d-array-distributed SYSTEM "./programs/Doof2d-Array-distributed-annotated.cpp">
    <!-- distributed Array stencil Doof2d implementation -->
! <!ENTITY doof2d-field-parallel SYSTEM "./programs/Doof2d-Field-parallel-annotated.cpp">
    <!-- Field data-parallel Doof2d implementation -->
! <!ENTITY doof2d-field-distributed SYSTEM "./programs/Doof2d-Field-distributed-annotated.cpp">
    <!-- Field data-parallel distributed Doof2d implementation -->
! <!ENTITY concepts-chapter SYSTEM "concepts.xml">
!   <!-- Pooma concepts chapter -->
! <!ENTITY glossary-chapter SYSTEM "glossary.xml">
!   <!-- glossary -->
! <!ENTITY tutorial-chapter SYSTEM "tutorial.xml">
!   <!-- Doof2d tutorial programs chapter -->
  ]>
  
  <book>
   <bookinfo>
    <title>&pooma;</title>
!   <subtitle>A &cc; Toolkit for High-Performance Parallel Scientific Computing</subtitle>
    <author><firstname>Jeffrey</firstname><othername
    role='mi'>D.</othername><surname>Oldham</surname>
     <affiliation>
--- 149,195 ----
    <!-- spelling: dependence, not dependency -->
    <!-- spelling: element-wise, not elementwise -->
    <!-- phrase: function object, not functor -->
+   <!-- spelling: interprocessor, not inter-processor -->
    <!-- spelling: multidimensional, not multi-dimensional -->
    <!-- spelling: multiprocessor, not multi-processor -->
    <!-- spelling: nonzero, not non-zero -->
  
+ <!-- External Chapters -->
+ <!ENTITY concepts-chapter SYSTEM "concepts.xml">
+   <!-- Pooma concepts chapter -->
+ <!ENTITY glossary-chapter SYSTEM "glossary.xml">
+   <!-- glossary -->
+ <!ENTITY introductory-chapter SYSTEM "introduction.xml">
+   <!-- Doof2d introductory chapter -->
+ <!ENTITY tutorial-chapter SYSTEM "tutorial.xml">
+   <!-- Doof2d tutorial programs chapter -->
+ 
  <!-- External File Entities -->
! <!-- Doof2d Programs -->
! <!ENTITY doof2d-c-element SYSTEM "./programs/examples/Doof2d/Doof2d-C-element-annotated.cpp">
    <!-- hand-coded Doof2d implementation -->
! <!ENTITY doof2d-array-element SYSTEM "./programs/examples/Doof2d/Doof2d-Array-element-annotated.cpp">
    <!-- Array element-wise Doof2d implementation -->
! <!ENTITY doof2d-array-parallel SYSTEM "./programs/examples/Doof2d/Doof2d-Array-parallel-annotated.cpp">
    <!-- Array data-parallel Doof2d implementation -->
! <!ENTITY doof2d-array-stencil SYSTEM "./programs/examples/Doof2d/Doof2d-Array-stencil-annotated.cpp">
    <!-- Array stencil Doof2d implementation -->
! <!ENTITY doof2d-array-distributed SYSTEM "./programs/examples/Doof2d/Doof2d-Array-distributed-annotated.cpp">
    <!-- distributed Array stencil Doof2d implementation -->
! <!ENTITY doof2d-field-parallel SYSTEM "./programs/examples/Doof2d/Doof2d-Field-parallel-annotated.cpp">
    <!-- Field data-parallel Doof2d implementation -->
! <!ENTITY doof2d-field-distributed SYSTEM "./programs/examples/Doof2d/Doof2d-Field-distributed-annotated.cpp">
    <!-- Field data-parallel distributed Doof2d implementation -->
! 
! <!-- Sequential Programs -->
! <!ENTITY initialize-finalize SYSTEM "./programs/examples/Sequential/initialize-finalize-annotated.cpp">
!   <!-- illustrate initialize() and finalize() -->
  ]>
  
  <book>
   <bookinfo>
    <title>&pooma;</title>
!   <subtitle>A &cc; &toolkitcap; for High-Performance Parallel Scientific Computing</subtitle>
    <author><firstname>Jeffrey</firstname><othername
    role='mi'>D.</othername><surname>Oldham</surname>
     <affiliation>
***************
*** 254,353 ****
  
   <part id="programming">
    <title>Programming with &pooma;</title>
- 
-   <chapter id="introduction">
-    <title>Introduction</title>
- 
-    <para>QUESTION: Add a partintro to the part above?</para>
- 
-    <para>&pooma; abbreviates <quote>Parallel Object-Oriented Methods
-    and Application</quote>.</para>
  
!    <para>This document is an introduction to &pooma; v2.1, a &cc;
!    toolkit for high-performance scientific computation.  &pooma;
!    runs efficiently on single-processor desktop machines,
!    shared-memory multiprocessors, and parallel supercomputers
!    containing dozens or hundreds of processors. What's more, by making
!    extensive use of the advanced features of the ANSI/ISO &cc;
!    standard&mdash;particularly templates&mdash;&pooma; presents a
!    compact, easy-to-read interface to its users.</para>
! 
!    <para>From Section&nbsp; of
!    <filename>papers/iscope98.pdf</filename>:</para>
! 
!    <para>Scientific software developers have struggled with the need
!    to express mathematical abstractions in an elegant and maintainable
!    way without sacrificing performance.  The &pooma; (Parallel
!    Object-Oriented Methods and Applications) framework, written in
!    <acronym>ANSI</acronym>/<acronym>ISO</acronym> &cc;, has
!    demonstrated both high expressiveness and high performance for
!    large-scale scientific applications on platforms ranging from
!    workstations to massively parallel supercomputers.  &pooma; provides
!    high-level abstractions for multidimensional arrays, physical
!    meshes, mathematical fields, and sets of particles.  &pooma; also
!    exploits techniques such as expression templates to optimize serial
!    performance while encapsulating the details of parallel
!    communication and supporting block-based data compression.
!    Consequently, scientists can quickly assemble parallel simulation
!    codes by focusing directly on the physical abstractions relevant to
!    the system under study and not the technical difficulties of
!    parallel communication and machine-specific optimization.</para>
! 
!    <para>ADD: diagram of science and &pooma;.  See the diagram that
!    Mark and I wrote.</para>
! 
!    <para>Mention efficient evaluation of &pooma; expressions.  See
!    <filename>pooma-publications/iscope98.pdf</filename>,
!    Section&nbsp;4.</para>
! 
!    <section id="introduction-pooma_evolution">
!     <title>Evolution of &pooma;</title>
! 
!     <para>QUESTION: Is this interesting?  Even if it is, it should be
!     short.</para>
! 
!     <para>The file <filename>papers/SCPaper-95.html</filename>
!     describes ?&pooma;1? and its abstraction layers.</para>
! 
!     <para>The "Introduction" of
!     <filename>papers/Siam0098.ps</filename> describes the DoE's
!     funding motivation for &pooma;: Accelerated Strategic Computing
!     Initiative (ASCI) and Science-based Stockpile Stewardship (SBSS),
!     pp. 1&ndash;2.</para>
! 
!     <para>See list of developers on p.&nbsp;1 of
!     <filename>papers/pooma.ps</filename>.</para>
! 
!     <para>See list of developers on p.&nbsp;1 of
!     <filename>papers/pooma.ps</filename>.  See history and motivation
!     on p.&nbsp;3 of <filename>papers/pooma.ps</filename>.</para>
  
!     <para>Use <filename class="libraryfile">README</filename> for
!     information.</para>
  
-     <blockquote>
-      <attribution><filename
- 			    class="libraryfile">introduction.html</filename></attribution>
- 
-      <para>&pooma; was designed and implemented by scientists working
-      at the Los Alamos National Laboratory's Advanced Computing
-      Laboratory. Between them, these scientists have written and tuned
-      large applications on almost every commercial and experimental
-      supercomputer built in the last two decades. As the technology
-      used in those machines migrates down into departmental computing
-      servers and desktop multiprocessors, &pooma; is a vehicle for its
-      designers' experience to migrate as well. In particular,
-      &pooma;'s authors understand how to get good performance out of
-      modern architectures, with their many processors and multi-level
-      memory hierarchies, and how to handle the subtly complex problems
-      that arise in real-world applications.</para>
-     </blockquote>
- 
-    </section>
- 
-   </chapter>
- 
- 
    &tutorial-chapter;
  
    &concepts-chapter;
--- 278,288 ----
  
   <part id="programming">
    <title>Programming with &pooma;</title>
  
! <!-- FIXME: Add a partintro to the part above? -->
  
!   &introductory-chapter;
  
    &tutorial-chapter;
  
    &concepts-chapter;
***************
*** 356,361 ****
--- 291,305 ----
    <chapter id="sequential">
     <title>Writing Sequential Programs</title>
  
+    <para>FIXME: Explain the chapter's purpose.
+ HERE</para>
+ 
+    <para>FIXME: Explain the format of each section.
+ HERE</para>
+ 
+    <para>FIXME: Explain the order  of the sections.
+ HERE</para>
+ 
     <para>Proposed order.  Basically follow the order in the proposed
     reference section.
      <orderedlist>
***************
*** 373,380 ****
      </orderedlist>
      Include views of containers in the appropriate sections.</para>
  
- <!-- HERE -->
- 
     <para><emphasis>&c;: A Reference Manual</emphasis> uses this
     structure for &c; libraries:
      <orderedlist>
--- 317,322 ----
***************
*** 524,555 ****
       </listitem>
      </orderedlist>
     </para>
  
!    <para>&pooma; can reorder computations to permit more efficient
!    computation.  When running a sequential program, reordering may
!    permit omission of unneeded computations.  For example, if only
!    values from a particular field are printed, only computations
!    involving the field and containers dependent on it need to occur.
!    When running a distributed program, reordering may permit
!    computation and communication among processors to overlap.  &pooma;
!    automatically tracks dependences between data-parallel expressions,
!    ensuring correct ordering.  It does not track statements accessing
!    particular &array; and &field; values so the programmer must
!    precede these statements with calls to
!    <function>Pooma::blockAndEvaluate()</function>.  Each call forces
!    the executable to wait until all computation has completed.  Thus,
!    the desired values are known to be available.  In practice, some
!    calls to <function>Pooma::blockAndEvaluate</function> may not be
!    necessary, but omitting them requires knowledge of &pooma;'s
!    dependence computations, so the &author; recommends calling
!    <function>Pooma::blockAndEvaluate</function> before each access to
!    a particular value in an &array; or &field;.  Omitting a necessary
!    call may lead to a race condition.  See <xref
!    linkend="debugging_profiling-missing_blockandevaluate"></xref> for
!    instructions how to diagnose and eliminate these race conditions.</para>
  
     <para>UNFINISHED</para>
  
     <section id="sequential-benchmarks">
      <title>&benchmark; Programs</title>
  
--- 466,663 ----
       </listitem>
      </orderedlist>
     </para>
+ 
+    <section id="sequential-begin_end">
+     <title>Beginning and Ending &pooma; Programs</title>
+ 
+     <para>Every &pooma; program must begin with a call to
+     <function>initialize</function> and end with a call to
+     <function>finalize</function>.  These functions respectively
+     prepare and shut down &pooma;'s run-time structures.</para>
+ 
+     <section id="sequential-begin_end-files">
+      <title>Files</title>
+ 
+      <programlisting>
+      #include "Pooma/Pooma.h"  // or "Pooma/Arrays.h" or "Pooma/Fields.h" or ...
+      </programlisting>
+     </section>
+ 
+     <section id="sequential-begin_end-declarations">
+       <title>Declarations</title>
+ 
+      <funcsynopsis>
+       <funcprototype>
+        <funcdef>bool <function>Pooma::initialize</function></funcdef>
+        <paramdef>
+         <parameter class="function">int &amp;argc,</parameter>
+         <parameter class="function">char ** &amp;argv,</parameter>
+         <parameter class="function">bool initRTS = true,</parameter>
+         <parameter class="function">bool getCLArgsArch = true,</parameter>
+         <parameter class="function">bool initArch = true</parameter>
+        </paramdef>
+       </funcprototype>
+ 
+       <funcprototype>
+        <funcdef>bool <function>Pooma::initialize</function></funcdef>
+        <paramdef>
+         <parameter class="function">Pooma::Options &amp;opts,</parameter>
+         <parameter class="function">bool initRTS = true,</parameter>
+         <parameter class="function">bool initArch = true</parameter>
+        </paramdef>
+       </funcprototype>
+ 
+       <funcprototype>
+        <funcdef>bool <function>Pooma::finalize</function></funcdef>
+        <void></void>
+       </funcprototype>
+ 
+       <funcprototype>
+        <funcdef>bool <function>Pooma::finalize</function></funcdef>
+        <paramdef>
+         <parameter class="function">bool quitRTS,</parameter>
+         <parameter class="function">bool quitArch</parameter>
+        </paramdef>
+       </funcprototype>
+      </funcsynopsis>
+     </section>
+ 
+     <section id="sequential-begin_end-description">
+      <title>Description</title>
+ 
+      <para>Before its use, the &poomatoolkit; must be initialized by a
+      call to <function>initialize</function>.  This usually occurs in
+      the <function>main</function> function.  The first form removes
+      and processes any &pooma;-specific arguments from the
+      command-line arguments <varname>argv</varname> and
+      <varname>argc</varname>.  <xref
+      linkend="sequential-options"></xref> describes these options.
+      The third, fourth, and fifth arguments all have a default value
+      of &true;.  If <parameter class="function">initRTS</parameter> is
+      &true;, the run-time system is initialized.  E.g., the contexts
+      are prepared for use.  If <parameter
+      class="function">getCLArgsArch</parameter> is &true,
+      architecture-specific command-line arguments are removed from
+      <varname>argv</varname> and <varname>argc</varname>.
+      Architecture-specific initialization occurs if <parameter
+      class="function">getCLArgsArch</parameter> is &true;.  An <link
+      linkend="glossary-architecture">architecture</link> is specified
+      by a hardware interface, e.g., processor type, but frequently is
+      also associated with an operating system or compiler.  For
+      example, Metrowerks for the Macintosh has an
+      architecture-specific initialization.  The function always
+      returns &true;.</para>
+ 
+      <para><function>initialize</function>'s alternative form
+      assumes the &pooma;-specific and architecture-specific
+      command-line arguments have already been removed from
+      <varname>argv</varname> and <varname>argc</varname> and stored in
+      <parameter class="function">opts</parameter>.  Its other two
+      parameters have the same meaning, and the two functions'
+      semantics are otherwise the same.</para>
+ 
+      <para>After its use, the &poomatoolkit; should be shut down using
+      a call to <function>finalize</function>.  This usually occurs in
+      the <function>main</function> function.  The former, and more
+      frequently used, form first prints any statistics and turns off
+      all default &pooma; streams.  Then it shuts down the run-time
+      system if it was previously initialized and then shuts down
+      architecture-specific objects if they were previously
+      initialized.  The latter form gives provides explicit control
+      whether the run-time system (<parameter
+      class="function">quitRTS</parameter>) and architecture-specific
+      objects (<parameter class="function">quitArch</parameter>) are
+      shut down.  Both functions always returns &true;.</para>
+ 
+      <para>Including almost any &pooma; header file, rather than just
+      <filename class="headerfile">Pooma/Pooma.h</filename> suffices
+      since most other &pooma; header files include it.</para>
+     </section>
+ 
+     <section id="sequential-begin_end-example">
+      <title>Example Program</title>
+ 
+      <para>Since every &pooma; program must call
+      <function>initialize</function> and
+      <function>finalize</function>, the simplest &pooma; program also
+      must call them.  This program also illustrates their usual
+      use.</para>
+ 
+      &initialize-finalize;
+     </section>
+ 
+    </section><!-- end sequential-begin_end -->
+ 
+    <section id="sequential-options">
+     <title>&pooma; Command-line Options</title>
+ 
+     <para>Every &pooma; program accepts a set of &pooma;-specific
+     command-line options to set values at run-time.</para>
+ 
+     <section id="sequential-options-list">
+      <title>Options Summary</title>
+ 
+      <variablelist>
+       <varlistentry>
+        <term><parameter class="option">&dashdash;pooma-info</parameter></term>
+        <listitem>
+ 	<para>
+ HERE  Who uses this?</para>
+        </listitem>
+       </varlistentry>
+ <!-- HERE -->
+      </variablelist>
  
!      <para>FIXME: Be sure to list default values.</para>
! 
! <!-- HERE -->
! 
!     </section>
! 
! <!-- HERE -->
! 
!     <para>QUESTION: Should I defer documenting &options; to the
!     reference manual, instead just listing commonly used options in
!     the previous section?
! 
! UNFINISHED</para>
! 
!    </section><!-- end sequential-options -->
! 
!    <section>
!     <title>TMP: Place these somewhere.</title>
  
+     <para>&pooma; can reorder computations to permit more efficient
+     computation.  When running a sequential program, reordering may
+     permit omission of unneeded computations.  For example, if only
+     values from a particular field are printed, only computations
+     involving the field and containers dependent on it need to occur.
+     When running a distributed program, reordering may permit
+     computation and communication among processors to overlap.
+     &pooma; automatically tracks dependences between data-parallel
+     expressions, ensuring correct ordering.  It does not track
+     statements accessing particular &array; and &field; values so the
+     programmer must precede these statements with calls to
+     <function>Pooma::blockAndEvaluate()</function>.  Each call forces
+     the executable to wait until all computation has completed.  Thus,
+     the desired values are known to be available.  In practice, some
+     calls to <function>Pooma::blockAndEvaluate</function> may not be
+     necessary, but omitting them requires knowledge of &pooma;'s
+     dependence computations, so the &author; recommends calling
+     <function>Pooma::blockAndEvaluate</function> before each access to
+     a particular value in an &array; or &field;.  Omitting a necessary
+     call may lead to a race condition.  See <xref
+     linkend="debugging_profiling-missing_blockandevaluate"></xref> for
+     instructions how to diagnose and eliminate these race
+     conditions.</para>
+ 
+     <para>Where talk about various &pooma; streams?</para>
+ 
     <para>UNFINISHED</para>
  
+    </section>
+ 
+ 
     <section id="sequential-benchmarks">
      <title>&benchmark; Programs</title>
  
***************
*** 561,573 ****
     </section>
  
  
-    <section id="sequential-inform">
-     <title>Using <type>Inform</type>s for Output</title>
- 
-     <para>UNFINISHED</para>
-    </section>
- 
- 
     <section>
      <title>Miscellaneous</title>
  
--- 669,674 ----
***************
*** 604,610 ****
      &pooma;&nbsp;II's expression trees and expression engines.</para>
  
      <para>COMMENT: <filename
! 			     class="libraryfile">background.html</filename> has some related
      &pete; material.</para>
     </section>
  
--- 705,711 ----
      &pooma;&nbsp;II's expression trees and expression engines.</para>
  
      <para>COMMENT: <filename
!     class="libraryfile">background.html</filename> has some related
      &pete; material.</para>
     </section>
  
***************
*** 652,659 ****
        in the input domain: A(i1, i2, ..., iN).</para>
  
       <para>The &pooma; multi-dimensional Array concept is similar to
!       the Fortran 90 array facility, but extends it in several
!       ways. Both &pooma; and Fortran arrays can have up to seven
        dimensions, and can serve as containers for arbitrary
        types. Both support the notion of views of a portion of the
        array, known as array sections in F90. The &pooma; Array concept
--- 753,760 ----
        in the input domain: A(i1, i2, ..., iN).</para>
  
       <para>The &pooma; multi-dimensional Array concept is similar to
!       the &fortran; 90 array facility, but extends it in several
!       ways. Both &pooma; and &fortran; arrays can have up to seven
        dimensions, and can serve as containers for arbitrary
        types. Both support the notion of views of a portion of the
        array, known as array sections in F90. The &pooma; Array concept
***************
*** 664,673 ****
        depending on the particular type of the Array being
        indexed.</para>
  
!      <para>Fortran arrays are dense and the elements are arranged
  		   according to column-major conventions. Therefore, X(i1,i2)
        refers to element number i1-1+(i2-1)*numberRowsInA. However, as
!       Fig. 1 shows, Fortran-style "Brick" storage is not the only
        storage format of interest to scientific programmers. For
        compatibility with C conventions, one might want to use an array
        featuring dense, row-major storage (a C-style Brick). To save
--- 765,774 ----
        depending on the particular type of the Array being
        indexed.</para>
  
!      <para>&fortran; arrays are dense and the elements are arranged
  		   according to column-major conventions. Therefore, X(i1,i2)
        refers to element number i1-1+(i2-1)*numberRowsInA. However, as
!       Fig. 1 shows, &fortran;-style "Brick" storage is not the only
        storage format of interest to scientific programmers. For
        compatibility with C conventions, one might want to use an array
        featuring dense, row-major storage (a C-style Brick). To save
***************
*** 691,697 ****
       themselves in the template parameters for the &pooma; Array
       class. The template
       <programlisting>
! 		      template &lt;int Dim, class T = double, class EngineTag = Brick&gt;
       class Array;
       </programlisting>
       is a specification for creating a set of classes all named
--- 792,798 ----
       themselves in the template parameters for the &pooma; Array
       class. The template
       <programlisting>
!      template &lt;int Dim, class T = double, class EngineTag = Brick&gt;
       class Array;
       </programlisting>
       is a specification for creating a set of classes all named
***************
*** 771,777 ****
      general Engine template whose template parameters are identical to
      those of Array. Next, the Array template determines the type of
      scalar arguments (indices) to be used in operator(), the function
!     that implements &pooma;'s Fortran-style indexing syntax X(i1,i2):
      <programlisting>
      typedef typename Engine_t::Index_t Index_t;
      </programlisting>
--- 872,878 ----
      general Engine template whose template parameters are identical to
      those of Array. Next, the Array template determines the type of
      scalar arguments (indices) to be used in operator(), the function
!     that implements &pooma;'s &fortran;-style indexing syntax X(i1,i2):
      <programlisting>
      typedef typename Engine_t::Index_t Index_t;
      </programlisting>
***************
*** 816,822 ****
      framework.</para>
  
      <para>Figure 3 illustrates the "Brick" specialization of the
!     Engine template, which implements Fortran-style lookup into a
      block of memory. First, there is the general Engine template,
      which is empty as there is no default behavior for an unknown
      EngineTag. The general template is therefore not a model for the
--- 917,923 ----
      framework.</para>
  
      <para>Figure 3 illustrates the "Brick" specialization of the
!     Engine template, which implements &fortran;-style lookup into a
      block of memory. First, there is the general Engine template,
      which is empty as there is no default behavior for an unknown
      EngineTag. The general template is therefore not a model for the
***************
*** 826,832 ****
      specialization of the Engine template. Finally, there is the
      partial specialization of the Engine template. Examining its body,
      we see the required Index_t typedef and the required operator(),
!     which follows the Fortran prescription for generating an offset
      into the data block based on the row, column, and the number of
      rows. All of the requirements are met, so the Brick-Engine class
      is a model of the Engine concept.</para>
--- 927,933 ----
      specialization of the Engine template. Finally, there is the
      partial specialization of the Engine template. Examining its body,
      we see the required Index_t typedef and the required operator(),
!     which follows the &fortran; prescription for generating an offset
      into the data block based on the row, column, and the number of
      rows. All of the requirements are met, so the Brick-Engine class
      is a model of the Engine concept.</para>
***************
*** 1899,1904 ****
--- 2000,2023 ----
      <title>TMP: What do we do with these &hellip;? Remove this
      section.</title>
  
+     <blockquote>
+      <attribution><filename
+       class="libraryfile">introduction.html</filename></attribution>
+ 
+      <para>&pooma; was designed and implemented by scientists working
+      at the Los Alamos National Laboratory's Advanced Computing
+      Laboratory. Between them, these scientists have written and tuned
+      large applications on almost every commercial and experimental
+      supercomputer built in the last two decades. As the technology
+      used in those machines migrates down into departmental computing
+      servers and desktop multiprocessors, &pooma; is a vehicle for its
+      designers' experience to migrate as well. In particular,
+      &pooma;'s authors understand how to get good performance out of
+      modern architectures, with their many processors and multi-level
+      memory hierarchies, and how to handle the subtly complex problems
+      that arise in real-world applications.</para>
+     </blockquote>
+ 
      <para>QUESTION: Do we describe the &leaffunctor;s specialized for
      &array;s in <filename
      class="headerfile">src/Array/Array.h</filename> or in the &pete;
***************
*** 2879,2898 ****
    </chapter>
  
  
!   <chapter id="where-place-these_ref">
     <title>TMP: Where do we describe these files?</title>
  
     <itemizedlist>
      <listitem>
       <para><filename
! 		     class="headerfile">src/Utilities/Conform.h</filename>: tag for
       checking whether terms in expression have conforming
       domains</para>
      </listitem>
  
      <listitem>
       <para><filename
! 		     class="headerfile">src/Utilities/DerefIterator.h</filename>:
       <type>DerefIterator&lt;T&gt;</type> and
       <type>ConstDerefIterator&lt;T&gt;</type> automatically
       dereference themselves to maintain <literal>const</literal>
--- 2998,3017 ----
    </chapter>
  
  
!   <chapter id="where_place_these_ref">
     <title>TMP: Where do we describe these files?</title>
  
     <itemizedlist>
      <listitem>
       <para><filename
!      class="headerfile">src/Utilities/Conform.h</filename>: tag for
       checking whether terms in expression have conforming
       domains</para>
      </listitem>
  
      <listitem>
       <para><filename
!      class="headerfile">src/Utilities/DerefIterator.h</filename>:
       <type>DerefIterator&lt;T&gt;</type> and
       <type>ConstDerefIterator&lt;T&gt;</type> automatically
       dereference themselves to maintain <literal>const</literal>
***************
*** 2901,2910 ****
  
      <listitem>
       <para><filename
! 		     class="headerfile">src/Utilities/Observable.h</filename>,
       <filename class="headerfile">src/Utilities/Observer.h</filename>,
       and <filename
! 		   class="headerfile">src/Utilities/ObserverEvent.h</filename>:
       <type>Observable&lt;T&gt;</type>,
       <type>SingleObserveable&lt;T&gt;</type>,
       <type>Observer&lt;T&gt;</type>, and <type>ObserverEvent</type>
--- 3020,3029 ----
  
      <listitem>
       <para><filename
!      class="headerfile">src/Utilities/Observable.h</filename>,
       <filename class="headerfile">src/Utilities/Observer.h</filename>,
       and <filename
!      class="headerfile">src/Utilities/ObserverEvent.h</filename>:
       <type>Observable&lt;T&gt;</type>,
       <type>SingleObserveable&lt;T&gt;</type>,
       <type>Observer&lt;T&gt;</type>, and <type>ObserverEvent</type>
***************
*** 2915,2920 ****
--- 3034,3053 ----
  
    </chapter>
  
+ 
+   <chapter id="needed_reference_items_ref">
+    <title>TMP: Items to Discuss in Reference Manual</title>
+ 
+    <itemizedlist>
+     <listitem>
+      <para>Discuss &options; and related material.  Add developer
+      command-line options listed in <filename
+      class="library">Utilities/Options.cmpl.cpp</filename> and also
+      possibly <parameter class="option">&dashdash;pooma-threads
+      <replaceable>n</replaceable></parameter>.</para>
+     </listitem>
+    </itemizedlist>
+   </chapter>
   </part>
  
  
***************
*** 2946,2952 ****
  
      <para>Section&nbsp;3, "Sample Applications" of
      <filename>papers/SiamOO98_paper.ps</filename> describes porting a
!     particle program written using High-Performance Fortran to
      &pooma; and presumably why particles were added to &pooma;.  It
      also describes <application>MC++</application>, a Monte Carlo
      neutron transport code.</para>
--- 3079,3085 ----
  
      <para>Section&nbsp;3, "Sample Applications" of
      <filename>papers/SiamOO98_paper.ps</filename> describes porting a
!     particle program written using High-Performance &fortran; to
      &pooma; and presumably why particles were added to &pooma;.  It
      also describes <application>MC++</application>, a Monte Carlo
      neutron transport code.</para>
***************
*** 3332,3338 ****
  
      <listitem>
       <para>QUESTION: How do &pooma; parallel concepts compare with
!      Fortran&nbsp;D or high-performance Fortran FINISH CITE:
       {koelbel94:_high_perfor_fortr_handb}?</para>
      </listitem>
  
--- 3465,3471 ----
  
      <listitem>
       <para>QUESTION: How do &pooma; parallel concepts compare with
!      &fortran;&nbsp;D or high-performance &fortran; FINISH CITE:
       {koelbel94:_high_perfor_fortr_handb}?</para>
      </listitem>
  
***************
*** 3500,3505 ****
--- 3633,3856 ----
     <title>Using MPI</title>
     <subtitle>Portable Parallel Programming with the Message-Passing Interface</subtitle>
     <edition>second edition</edition>
+   </biblioentry>
+ 
+   <biblioentry>
+    <abbrev>pooma95</abbrev>
+    <authorgroup>
+     <author>
+      <firstname>John</firstname><othername role="mi">V. W.</othername><surname>Reynders</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Paul</firstname><othername role="mi">J.</othername><surname>Hinker</surname>
+      <affiliation>
+       <orgname>Dakota Software Systems, Inc.</orgname>
+       <address><city>Rapid City</city><state>SD</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Julian</firstname><othername role="mi">C.</othername><surname>Cummings</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Susan</firstname><othername role="mi">R.</othername><surname>Atlas</surname>
+      <affiliation>
+       <orgname>Parallel Solutions, Inc.</orgname>
+       <address><city>Santa Fe</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Subhankar</firstname><surname>Banerjee</surname>
+      <affiliation>
+       <orgname>New Mexico State University</orgname>
+       <address><city>Las Cruces</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>William</firstname><othername role="mi">F.</othername><surname>Humphrey</surname>
+      <affiliation>
+       <orgname>University of Illinois at Urbana-Champaign</orgname>
+       <address><city>Urbana-Champaign</city><state>IL</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Steve</firstname><othername role="mi">R.</othername><surname>Karmesin</surname>
+      <affiliation>
+       <orgname>California Institute of Technology</orgname>
+       <address><city>Pasadena</city><state>CA</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Katarzyna</firstname><surname>Keahey</surname>
+      <affiliation>
+       <orgname>Indiana University</orgname>
+       <address><city>Bloomington</city><state>IN</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Marydell</firstname><surname>Tholburn</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+    </authorgroup>
+    <title>&pooma;</title>
+    <subtitle>A Framework for Scientific Simulation on Parallel Architectures</subtitle>
+    <releaseinfo>unpublished</releaseinfo>
+   </biblioentry>
+ 
+   <biblioentry>
+    <abbrev>pooma-sc95</abbrev>
+    <authorgroup>
+     <author>
+      <firstname>Susan</firstname><surname>Atlas</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Subhankar</firstname><surname>Banerjee</surname>
+      <affiliation>
+       <orgname>New Mexico State University</orgname>
+       <address><city>Las Cruces</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Julian</firstname><othername role="mi">C.</othername><surname>Cummings</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Paul</firstname><othername role="mi">J.</othername><surname>Hinker</surname>
+      <affiliation>
+       <orgname>Advanced Computing Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>M.</firstname><surname>Srikant</surname>
+      <affiliation>
+       <orgname>New Mexico State University</orgname>
+       <address><city>Las Cruces</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>John</firstname><othername role="mi">V. W.</othername><surname>Reynders</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Marydell</firstname><surname>Tholburn</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+    </authorgroup>
+    <title>&pooma;</title>
+    <subtitle>A High Performance Distributed Simulation Environment for
+    Scientific Applications</subtitle>
+ <!-- FIXME: Where list Supercomputing 1995? -->
+   </biblioentry>
+ 
+   <biblioentry>
+    <abbrev>pooma-siam98</abbrev>
+    <authorgroup>
+     <author>
+      <firstname>Julian</firstname><othername role="mi">C.</othername><surname>Cummings</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>James</firstname><othername role="mi">A.</othername><surname>Crotinger</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Scott</firstname><othername role="mi">W.</othername><surname>Haney</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>William</firstname><othername role="mi">F.</othername><surname>Humphrey</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Steve</firstname><othername role="mi">R.</othername><surname>Karmesin</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>John</firstname><othername role="mi">V. W.</othername><surname>Reynders</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Stephen</firstname><othername role="mi">A.</othername><surname>Smith</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+     <author>
+      <firstname>Timothy</firstname><othername role="mi">J.</othername><surname>Williams</surname>
+      <affiliation>
+       <orgname>Los Alamos National Laboratory</orgname>
+       <address><city>Los Alamos</city><state>NM</state></address>
+      </affiliation>
+     </author>
+    </authorgroup>
+    <title>Raid Application Development and Enhanced Code
+    Interoperability using the &pooma; Framework</title>
+ <!-- FIXME: Where list SIAM Workshop ... 1998? -->
+   </biblioentry>
+ 
+   <biblioentry>
+ <!-- FIXME: Change the year when we learn it. -->
+    <abbrev>pete-99</abbrev>
+    <authorgroup>
+     <author>
+      <firstname>Scott</firstname><surname>Haney</surname>
+     </author>
+     <author>
+      <firstname>James</firstname><surname>Crotinger</surname>
+     </author>
+     <author>
+      <firstname>Steve</firstname><surname>Karmesin</surname>
+     </author>
+     <author>
+      <firstname>Stephen</firstname><surname>Smith</surname>
+     </author>
+    </authorgroup>
+    <title>Easy Expression Templates Using &pete;: The Portable
+    Expression Template Engine</title>
+ <!-- FIXME: When and where was this published? -->
    </biblioentry>
   </bibliography>
  
Index: tutorial.xml
===================================================================
RCS file: /home/pooma/Repository/r2/docs/manual/tutorial.xml,v
retrieving revision 1.2
diff -c -p -r1.2 tutorial.xml
*** tutorial.xml	2001/12/14 04:18:13	1.2
--- tutorial.xml	2001/12/17 16:56:55
***************
*** 36,42 ****
     </listitem>
     <listitem>
      <para>a data-parallel &pooma; &field; implementation for
!     multi-processor execution.</para>
     </listitem>
    </itemizedlist>
   </para>
--- 36,42 ----
     </listitem>
     <listitem>
      <para>a data-parallel &pooma; &field; implementation for
!     multiprocessor execution.</para>
     </listitem>
    </itemizedlist>
   </para>
***************
*** 94,100 ****
   zero.</para>
  
   <para>Before presenting various implementations of %doof2d;, we
!  explain how to install the &poomaToolkit;.</para>
  
   <para>REMOVE: &doof2d; algorithm and code is illustrated in
   Section&nbsp;4.1 of
--- 94,100 ----
   zero.</para>
  
   <para>Before presenting various implementations of %doof2d;, we
!  explain how to install the &poomatoolkit;.</para>
  
   <para>REMOVE: &doof2d; algorithm and code is illustrated in
   Section&nbsp;4.1 of
***************
*** 111,117 ****
    <quote>LINUXgcc.conf</quote> is not available.</para>
  
    <para>In this section, we describe how to obtain, build, and
!   install the &poomaToolkit;.  We focus on installing under the
    Unix operating system.  Instructions for installing on computers
    running Microsoft Windows or MacOS, as well as more extensive
    instructions for Unix, appear in <xref
--- 111,117 ----
    <quote>LINUXgcc.conf</quote> is not available.</para>
  
    <para>In this section, we describe how to obtain, build, and
!   install the &poomatoolkit;.  We focus on installing under the
    Unix operating system.  Instructions for installing on computers
    running Microsoft Windows or MacOS, as well as more extensive
    instructions for Unix, appear in <xref
***************
*** 142,148 ****
    <command>&dashdash;arch</command> option is the name of the corresponding
    configuration file, omitting its <filename
    class="libraryfile">.conf</filename> suffix.  The
!   <command>&dashdash;opt</command> indicates the &poomaToolkit; will
    contain optimized source code, which makes the code run more
    quickly but may impede debugging.  Alternatively, the
    <command>&dashdash;debug</command> option supports debugging.  The
--- 142,148 ----
    <command>&dashdash;arch</command> option is the name of the corresponding
    configuration file, omitting its <filename
    class="libraryfile">.conf</filename> suffix.  The
!   <command>&dashdash;opt</command> indicates the &poomatoolkit; will
    contain optimized source code, which makes the code run more
    quickly but may impede debugging.  Alternatively, the
    <command>&dashdash;debug</command> option supports debugging.  The
***************
*** 178,184 ****
   <section id="tutorial-hand_coded">
    <title>Hand-Coded Implementation</title>
  
!   <para>Before implementing &doof2d; using the &poomaToolkit;, we
    present a hand-coded implementation of &doof2d;.  See <xref
    linkend="tutorial-hand_coded-doof2d"></xref>.  After querying the
    user for the number of averagings, the arrays' memory is
--- 178,184 ----
   <section id="tutorial-hand_coded">
    <title>Hand-Coded Implementation</title>
  
!   <para>Before implementing &doof2d; using the &poomatoolkit;, we
    present a hand-coded implementation of &doof2d;.  See <xref
    linkend="tutorial-hand_coded-doof2d"></xref>.  After querying the
    user for the number of averagings, the arrays' memory is
***************
*** 290,296 ****
   <section id="tutorial-array_elementwise">
    <title>Element-wise &array; Implementation</title>
  
!   <para>The simplest way to use the &poomaToolkit; is to
    use the &pooma; &array; class instead of &c; arrays.  &array;s
    automatically handle memory allocation and deallocation, support a
    wider variety of assignments, and can be used in expressions.
--- 290,296 ----
   <section id="tutorial-array_elementwise">
    <title>Element-wise &array; Implementation</title>
  
!   <para>The simplest way to use the &poomatoolkit; is to
    use the &pooma; &array; class instead of &c; arrays.  &array;s
    automatically handle memory allocation and deallocation, support a
    wider variety of assignments, and can be used in expressions.
***************
*** 309,315 ****
       class="headerfile">Pooma/Arrays.h</filename> must be included.</para>
      </callout>
      <callout arearefs="tutorial-array_elementwise-doof2d-pooma_initialize">
!      <para>The &poomaToolkit; structures must be constructed before
       their use.</para>
      </callout>
      <callout arearefs="tutorial-array_elementwise-doof2d-domain">
--- 309,315 ----
       class="headerfile">Pooma/Arrays.h</filename> must be included.</para>
      </callout>
      <callout arearefs="tutorial-array_elementwise-doof2d-pooma_initialize">
!      <para>The &poomatoolkit; structures must be constructed before
       their use.</para>
      </callout>
      <callout arearefs="tutorial-array_elementwise-doof2d-domain">
***************
*** 347,359 ****
       memory leaks.</para>
      </callout>
      <callout arearefs="tutorial-array_elementwise-doof2d-pooma_finish">
!      <para>The &poomaToolkit; structures must be destructed after
       their use.</para>
      </callout>
     </calloutlist>
    </example>
  
!   <para>We describe the use of &array; and the &poomaToolkit; in
    <xref linkend="tutorial-array_elementwise-doof2d"></xref>.
    &array;s, declared in the <filename
    class="headerfile">Pooma/Arrays.h</filename>, are first-class
--- 347,359 ----
       memory leaks.</para>
      </callout>
      <callout arearefs="tutorial-array_elementwise-doof2d-pooma_finish">
!      <para>The &poomatoolkit; structures must be destructed after
       their use.</para>
      </callout>
     </calloutlist>
    </example>
  
!   <para>We describe the use of &array; and the &poomatoolkit; in
    <xref linkend="tutorial-array_elementwise-doof2d"></xref>.
    &array;s, declared in the <filename
    class="headerfile">Pooma/Arrays.h</filename>, are first-class
***************
*** 391,398 ****
    elements.  This is possible because the array knows the extent of
    its domain.</para>
  
!   <para>Any program using the &poomaToolkit; must initialize the
!   toolkit's data structures using
    <statement>Pooma::initialize(argc,argv)</statement>.  This
    extracts &pooma;-specific command-line options from the
    command-line arguments in <varname>argv</varname> and initializes
--- 391,398 ----
    elements.  This is possible because the array knows the extent of
    its domain.</para>
  
!   <para>Any program using the &poomatoolkit; must initialize the
!   &toolkit;'s data structures using
    <statement>Pooma::initialize(argc,argv)</statement>.  This
    extracts &pooma;-specific command-line options from the
    command-line arguments in <varname>argv</varname> and initializes
***************
*** 408,414 ****
  
    <para>&pooma; supports data-parallel &array; accesses.  Many
    algorithms are more easily expressed using data-parallel
!   expressions.  Also, the &poomaToolkit; might be able to reorder
    the data-parallel computations to be more efficient or distribute
    them among various processors.  In this section, we concentrate
    the differences between the data-parallel implementation of
--- 408,414 ----
  
    <para>&pooma; supports data-parallel &array; accesses.  Many
    algorithms are more easily expressed using data-parallel
!   expressions.  Also, the &poomatoolkit; might be able to reorder
    the data-parallel computations to be more efficient or distribute
    them among various processors.  In this section, we concentrate
    the differences between the data-parallel implementation of
***************
*** 618,624 ****
    indicates a particular dimension.  Index parameters
    <varname>i</varname> and <varname>j</varname> are in dimension 0
    and&nbsp;1.  <methodname>upperExtent</methodname> serves an
!   analogous purpose.  The &poomaToolkit; uses these functions when
    distribution computation among various processors, but it does not
    use these functions to ensure nonexistent &array; values are not
    accessed.  Caveat stencil user!</para>
--- 618,624 ----
    indicates a particular dimension.  Index parameters
    <varname>i</varname> and <varname>j</varname> are in dimension 0
    and&nbsp;1.  <methodname>upperExtent</methodname> serves an
!   analogous purpose.  The &poomatoolkit; uses these functions when
    distribution computation among various processors, but it does not
    use these functions to ensure nonexistent &array; values are not
    accessed.  Caveat stencil user!</para>
***************
*** 632,638 ****
    To convert a program designed for uniprocessor execution to a
    program designed for multiprocessor execution, the programmer need
    only specify how each container's domain should be split into
!   <quote>patches</quote>.  The &poomaToolkit; automatically
    distributes the data among the available processors and handles
    any required communication among processors.</para>
  
--- 632,638 ----
    To convert a program designed for uniprocessor execution to a
    program designed for multiprocessor execution, the programmer need
    only specify how each container's domain should be split into
!   <quote>patches</quote>.  The &poomatoolkit; automatically
    distributes the data among the available processors and handles
    any required communication among processors.</para>
  
***************
*** 746,761 ****
    configured for the particular run-time system.  See <xref
    linkend="installation-distributed_computing"></xref>.</para>
  
!   <para>A <firstterm>layout</firstterm> combines patches with
!   contexts so the program can be executed.  If &distributedtag; is
!   specified, the patches are distributed among the available
!   contexts.  If &replicatedtag; is specified, each set of patches is
!   replicated among each context.  Regardless, the containers'
!   domains are now distributed among the contexts so the program can
!   run.  When a patch needs data from another patch, the &pooma;
!   toolkit sends messages to the desired patch uses a message-passing
!   library.  All such communication is automatically performed by the
!   toolkit with no need for programmer or user input.</para>
  
    <para>FIXME: The two previous paragraphs demonstrate confusion
    between <quote>run-time system</quote> and <quote>message-passing
--- 746,761 ----
    configured for the particular run-time system.  See <xref
    linkend="installation-distributed_computing"></xref>.</para>
  
!   <para>A <firstterm>layout</firstterm> combines patches with contexts
!   so the program can be executed.  If &distributedtag; is specified,
!   the patches are distributed among the available contexts.  If
!   &replicatedtag; is specified, each set of patches is replicated
!   among each context.  Regardless, the containers' domains are now
!   distributed among the contexts so the program can run.  When a patch
!   needs data from another patch, the &poomatoolkit; sends messages to
!   the desired patch uses a message-passing library.  All such
!   communication is automatically performed by the &toolkit; with no need
!   for programmer or user input.</para>
  
    <para>FIXME: The two previous paragraphs demonstrate confusion
    between <quote>run-time system</quote> and <quote>message-passing
***************
*** 803,811 ****
    &gt;</type> or <type>MultiPatch&lt;UniformTag,
    Remote&lt;CompressibleBrick&gt; &gt;</type> engines.</para>
  
!   <para>The computations for a distributed implementation are
!   exactly the same as for a sequential implementation.  The &pooma;
!   Toolkit and a message-passing library automatically perform all
    computation.</para>
  
    <para>Input and output for distributed programs is different than
--- 803,811 ----
    &gt;</type> or <type>MultiPatch&lt;UniformTag,
    Remote&lt;CompressibleBrick&gt; &gt;</type> engines.</para>
  
!   <para>The computations for a distributed implementation are exactly
!   the same as for a sequential implementation.  The &poomatoolkit; and
!   a message-passing library automatically perform all
    computation.</para>
  
    <para>Input and output for distributed programs is different than
***************
*** 988,994 ****
      </callout>
      <callout arearefs="tutorial-field_distributed-doof2d-mesh">
       <para>The mesh and centering declarations are the same for
!      uniprocessor and multi-processor implementations.</para>
      </callout>
      <callout arearefs="tutorial-field_distributed-doof2d-remote">
       <para>The <type>MultiPatch</type> &engine; distributes requests
--- 988,994 ----
      </callout>
      <callout arearefs="tutorial-field_distributed-doof2d-mesh">
       <para>The mesh and centering declarations are the same for
!      uniprocessor and multiprocessor implementations.</para>
      </callout>
      <callout arearefs="tutorial-field_distributed-doof2d-remote">
       <para>The <type>MultiPatch</type> &engine; distributes requests
***************
*** 1032,1038 ****
      </listitem>
      <listitem>
       <para>The computation for uniprocessor or distributed
!      implementations remains the same.  The &pooma; toolkit
       automatically handles all communication necessary to ensure
       up-to-date values are available when needed.</para>
      </listitem>
--- 1032,1038 ----
      </listitem>
      <listitem>
       <para>The computation for uniprocessor or distributed
!      implementations remains the same.  The &poomatoolkit;
       automatically handles all communication necessary to ensure
       up-to-date values are available when needed.</para>
      </listitem>
Index: figures/introduction.mp
===================================================================
RCS file: introduction.mp
diff -N introduction.mp
*** /dev/null	Fri Mar 23 21:37:44 2001
--- introduction.mp	Mon Dec 17 09:56:55 2001
***************
*** 0 ****
--- 1,194 ----
+ %% Oldham, Jeffrey D.
+ %% 2001Dec14
+ %% Pooma
+ 
+ %% Illustrations for Introduction
+ 
+ %% Assumes TEX=latex.
+ 
+ input boxes;
+ 
+ verbatimtex
+ \documentclass[10pt]{article}
+ \usepackage{amsmath}
+ \input{macros.ltx}
+ \begin{document}
+ etex
+ 
+ %% Relationship between science, computational science, and Pooma.
+ beginfig(101)
+   numeric unit; unit = 0.8cm;
+   numeric horizSpace; horizSpace = 8unit;
+   numeric vertSpace; vertSpace = unit;
+   numeric nuBoxes;		% number of boxes
+ 
+   % Ensure a list of boxes all have the same width.
+   % input <- suffixes for the boxes;
+   % output-> all boxes have the same width (maximum picture width + defaultdx)
+   vardef samewidth(suffix $)(text t) =
+     save p_; pair p_;
+     p_ = maxWidthAndHeight($)(t);
+     numericSetWidth(xpart(p_)+2defaultdx)($)(t);
+   enddef;
+   
+   % Ensure a list of boxes all have the same height.
+   % input <- suffixes for the boxes;
+   % output-> all boxes have the same height (maximum picture height + defaultdy)
+   vardef sameheight(suffix $)(text t) =
+     save p_; pair p_;
+     p_ = maxWidthAndHeight($)(t);
+     numericSetWidth(ypart(p_)+2defaultdy)($)(t);
+   enddef;
+   
+   % Given a list of boxes, determine the maximum picture width and
+   % maximum picture height.
+   % input <- suffixes for the boxes
+   % output-> pair of maximum picture width and height
+   vardef maxWidthAndHeight(suffix f)(text t) =
+     save w_, h_; numeric w_, h_;
+     w_ = xpart((urcorner pic_.f - llcorner pic_.f));
+     h_ = ypart((urcorner pic_.f - llcorner pic_.f));
+     forsuffixes uu = t:
+       if xpart((urcorner pic_.uu - llcorner pic_.uu)) > w_ :
+ 	w_ := xpart((urcorner pic_.uu - llcorner pic_.uu));
+       fi
+       if ypart((urcorner pic_.uu - llcorner pic_.uu)) > h_ :
+ 	h_ := ypart((urcorner pic_.uu - llcorner pic_.uu));
+       fi
+     endfor
+     (w_, h_)
+   enddef;
+ 
+   % Given a width, ensure a box has the given width.
+   % input <- box width
+   %          suffix for the one box
+   % output-> the box has the given width by setting its .dx
+   vardef numericSetWidthOne(expr width)(suffix f) =
+     f.dx = 0.5(width - xpart(urcorner pic_.f - llcorner pic_.f));
+   enddef;
+   
+   % Given a width, ensure all boxes have the given width.
+   % input <- box width
+   %          suffixes for the boxes
+   % output-> all boxes have the given width by setting their .dx
+   vardef numericSetWidth(expr width)(suffix f)(text t) =
+     f.dx = 0.5(width - xpart(urcorner pic_.f - llcorner pic_.f));
+     forsuffixes $ = t:
+       $.dx = 0.5(width - xpart(urcorner pic_.$ - llcorner pic_.$));
+     endfor
+   enddef;
+ 
+   % Given a height, ensure all boxes have the given height.
+   % input <- box height
+   %          suffixes for the boxes
+   % output-> all boxes have the given height by setting their .dx
+   vardef numericSetHeight(expr height)(suffix f)(text t) =
+     f.dy = 0.5(height - ypart(urcorner pic_.f - llcorner pic_.f));
+     forsuffixes $ = t:
+       $.dy = 0.5(height - ypart(urcorner pic_.$ - llcorner pic_.$));
+     endfor
+   enddef;
+   
+   % Ensure a list of boxes and circles all to have the same width, height,
+   % and diameter.
+   % input <- suffixes for the boxes and circles
+   % output-> all boxes have .dx and .dy set so they have the same width,
+   %           height, and radius
+   % The boxes are squares and the circles are circular, not oval.
+   vardef sameWidthAndHeight(suffix f)(text t) =
+     save p_; pair p_;
+     p_ = maxWidthAndHeight(f)(t);
+     if (xpart(p_)+2defaultdx >= ypart(p_)+2defaultdy):
+       numericSetWidth(xpart(p_)+2defaultdx)(f)(t);
+       numericSetHeight(xpart(p_)+2defaultdx)(f)(t);
+     else:
+       numericSetWidth(ypart(p_)+2defaultdy)(f)(t);
+       numericSetHeight(ypart(p_)+2defaultdy)(f)(t);
+     fi
+   enddef;
+ 
+   % Ensure a list of boxes and circles all to have the same width and
+   % the same height.  Unlike sameWidthAndHeight, the width and height
+   % can differ.
+   % input <- suffixes for the boxes and circles
+   % output-> all boxes have .dx and .dy set so they have the same width,
+   %           height, and radius
+   % The boxes are squares and the circles are circular, not oval.
+   vardef sameWidthSameHeight(suffix f)(text t) =
+     save p_; pair p_;
+     p_ = maxWidthAndHeight(f)(t);
+     numericSetWidth(xpart(p_)+2defaultdx)(f)(t);
+     numericSetHeight(ypart(p_)+2defaultdy)(f)(t);
+   enddef;
+ 
+   % Create the boxes.
+   boxit.b0(btex \textsl{science / math} etex);
+   boxit.b1(btex \textsl{algorithms} etex);
+   boxit.b2(btex \textsl{engineering} etex);
+   boxit.b3(btex \strut $\real^{\dimension} \maps \text{values}$ etex);
+   boxit.b4(btex \strut $\text{discrete space} \maps \text{values}$ etex);
+   boxit.b5(btex \strut $(\text{layout}, \text{engine}) \maps \text{values}$ etex);
+   boxit.b6(btex \strut linear algebra etex);
+   boxit.b7(btex \strut $\naturalNus^{\dimension} \maps \text{values}$ etex);
+   boxit.b8(btex etex);
+   nuBoxes = 8;
+   boxit.b9(btex \textsl{implementation} etex);
+   sameWidthSameHeight(b3,b4,b5,b6,b7,b8);
+   for t = 0 upto nuBoxes+1:
+     fixsize(b[t]);
+   endfor
+   
+   % Position the boxes.
+   b0.c = origin;
+   for t = 0 step 3 until nuBoxes:
+     b[t+2].c - b[t+1].c = b[t+1].c - b[t].c = (horizSpace, 0);
+   endfor
+   for t = 0 step 3 until nuBoxes-3:
+     b[t].s - b[t+3].n = (0, vertSpace);
+   endfor
+   b9.c = 0.5[b1.c,b2.c];
+   
+   % Draw the boxes.
+   for t = 0 upto nuBoxes+1:
+     if unknown(b[t].c):
+       show t;
+       show b[t].c;
+     fi
+   endfor
+ 
+   for t = 0 upto 2:
+     drawunboxed(b[t]);
+   endfor
+   for t = 3 upto nuBoxes-1:
+     drawboxed(b[t]);
+   endfor
+   drawunboxed(b9);
+   
+   % Label the boxes.
+   label.top(btex continuous field etex, b3.n);
+   label.top(btex discrete field etex, b4.n);
+   label.top(btex \pooma\ container etex, b5.n);
+   label.top(btex mathematical array etex, b7.n);
+ %  label.top(btex custom implementation etex, b8.n);
+ 
+   % Draw the arrows.
+   vardef drawAndLabelArrow(expr start, stop, txt, parr) =
+     path p; p = start -- stop;
+     drawarrow p;
+     label.top(txt rotated angle (direction parr of p), point parr of p);
+   enddef;
+   vardef drawAndLabelArrowDashed(expr start, stop, txt, parr) =
+     path p; p = start -- stop;
+     drawarrow p dashed evenly;
+     label.top(txt rotated angle (direction parr of p), point parr of p);
+   enddef;
+ %  drawAndLabelArrowDashed(b4.e, b8.w, btex etex, 0.5);
+ %  drawAndLabelArrowDashed(b7.e, b8.w, btex etex, 0.5);
+   drawAndLabelArrow(b3.e, b4.w, btex discretization etex, 0.5);
+   drawAndLabelArrow(b4.e, b5.w, btex \type{Field} etex, 0.5);
+   drawAndLabelArrow(b6.e, b7.w, btex \begin{tabular}{c} numerical\\ analysis \end{tabular} etex, 0.5);
+   drawAndLabelArrow(b7.e, b5.w, btex \type{Array} etex, 0.3);
+   
+ endfig;
+ 
+ bye

From oldham at codesourcery.com  Tue Dec 18 00:50:38 2001
From: oldham at codesourcery.com (Jeffrey Oldham)
Date: Mon, 17 Dec 2001 16:50:38 -0800
Subject: Patch: Speed Pooma Evaluations
Message-ID: <20011217165038.A3289@codesourcery.com>

1. This patch permits the KCC optimizer to move most of non-data array
accesses out of the inner loop.  By using local variables rather than
references, the optimizer can determine that the inner loops'
assignments do not change the local objects' data members, e.g.,
strides_m.  Thus, these values need not be reloaded inside the inner
loops so the code is similar to hand-coded C loops.  In effect, the
compiler can determine which data members are loop-invariant and which
are not.  Example execution time (seconds) for Linux/KCC's Doof2d with
N=1000 include
 
                  C     Brick   FieldBrick stencil Brick
before change     6.20  9.89    13.29      7.29
after change      6.21  7.48    7.44       7.16
 
2. The idea can be implemented in at least two different ways.
Stephen Smith suggested the idea for the attached patch.  It relies on
two assumptions:
1) cheap, shallow copies and
2) copies of all non-pointer data members.

3. In Mark Mitchell's suggested implementation, container and engine
data members that are invariant during loop iterations are explicitly
stored in LoopInvariant_t structures.  These constant structures are
constructed before the loop and passed to the reads and writes within
the loop.  These operations use the constant structures rather than
the containers' and engines' data members.  Thus, the optimizer can
determine that the uses of the constant data members can be hoisted
out of the loops.  Although this implementation can deliver cleaner
code since we, as smart humans, might be able to determine better
code, it requires much more programmer time and code.  We can always
implement the idea if needed.  A patch for part of the work is
attached.

4. Two other sets of loops could be sped up using a similar technique
but were not.

a. Evaluator/LoopApply.h uses a function object.  Since we do not know
   whether we can copy the object much less copy back into the
   original, I do not know how to transform the loops.

b. Engine/RemoteEngine.h's EngineBlockSerialize could be modified but
   I could not find any user code to confirm the transformation's
   correctness.

Thanks to Mark Mitchell for finding the idea and creating the
technique.  Thanks to Stephen Smith for finding the slicker
implementation.

2001-11-02  Jeffrey D. Oldham  <oldham at codesourcery.com>

        * InlineEvaluator.h
        (KernelEvaluator<InlineKernelTag>::evaluate() for Dim=1..7:
        Use local variables for the left-hand side and the right-hand
        side.  This permits the KCC optimizer to move loop-invariant code
        out of the innermost loop, significantly reducing running times.
        * ReductionEvaluator.h
        (ReductionEvaluator<InlineKernelTag>::evaluate() for Dim=1..7:
        Use local variables for the expression and the accumulator
        variable.  This permits the KCC optimizer to move loop-invariant
        code out of the innermost loop, significantly reducing running
        times.

Tested on       Linux/KCC by compiling Doof2d and running all the
                array regression tests.  Only the inner loops of
                Doof2d and src/Evaluator/tests/ReductionTest1 were
                investigated.  (Doof2d compiled 17Dec using LINUXgcc
                --opt.)
Approved by     Stephen Smith
Applied to      mainline

Thanks,
Jeffrey D. Oldham
oldham at codesourcery.com
-------------- next part --------------
Index: InlineEvaluator.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Evaluator/InlineEvaluator.h,v
retrieving revision 1.26
diff -c -p -r1.26 InlineEvaluator.h
*** InlineEvaluator.h	2001/04/13 02:15:06	1.26
--- InlineEvaluator.h	2001/11/03 00:15:14
*************** struct KernelEvaluator<InlineKernelTag>
*** 157,165 ****
    {
      CTAssert(Domain::unitStride);
      PAssert(domain[0].first() == 0);
      int e0 = domain[0].length();
      for (int i0=0; i0<e0; ++i0)
!       op(lhs(i0),rhs.read(i0));
    }
  
    template<class LHS,class Op,class RHS,class Domain>
--- 157,167 ----
    {
      CTAssert(Domain::unitStride);
      PAssert(domain[0].first() == 0);
+     LHS localLHS(lhs);
+     RHS localRHS(rhs);
      int e0 = domain[0].length();
      for (int i0=0; i0<e0; ++i0)
!       op(localLHS(i0),localRHS.read(i0));
    }
  
    template<class LHS,class Op,class RHS,class Domain>
*************** struct KernelEvaluator<InlineKernelTag>
*** 169,179 ****
      CTAssert(Domain::unitStride);
      PAssert(domain[0].first() == 0);
      PAssert(domain[1].first() == 0);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      for (int i1=0; i1<e1; ++i1)
        for (int i0=0; i0<e0; ++i0)
! 	op(lhs(i0,i1),rhs.read(i0,i1));
    }
    
    template<class LHS,class Op,class RHS,class Domain>
--- 171,183 ----
      CTAssert(Domain::unitStride);
      PAssert(domain[0].first() == 0);
      PAssert(domain[1].first() == 0);
+     LHS localLHS(lhs);
+     RHS localRHS(rhs);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      for (int i1=0; i1<e1; ++i1)
        for (int i0=0; i0<e0; ++i0)
! 	op(localLHS(i0,i1),localRHS.read(i0,i1));
    }
    
    template<class LHS,class Op,class RHS,class Domain>
*************** struct KernelEvaluator<InlineKernelTag>
*** 184,196 ****
      PAssert(domain[0].first() == 0);
      PAssert(domain[1].first() == 0);
      PAssert(domain[2].first() == 0);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      int e2 = domain[2].length();
      for (int i2=0; i2<e2; ++i2)
        for (int i1=0; i1<e1; ++i1)
  	for (int i0=0; i0<e0; ++i0)
! 	  op(lhs(i0,i1,i2),rhs.read(i0,i1,i2));
    }
  
    template<class LHS,class Op,class RHS,class Domain>
--- 188,202 ----
      PAssert(domain[0].first() == 0);
      PAssert(domain[1].first() == 0);
      PAssert(domain[2].first() == 0);
+     LHS localLHS(lhs);
+     RHS localRHS(rhs);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      int e2 = domain[2].length();
      for (int i2=0; i2<e2; ++i2)
        for (int i1=0; i1<e1; ++i1)
  	for (int i0=0; i0<e0; ++i0)
! 	  op(localLHS(i0,i1,i2),localRHS.read(i0,i1,i2));
    }
  
    template<class LHS,class Op,class RHS,class Domain>
*************** struct KernelEvaluator<InlineKernelTag>
*** 202,207 ****
--- 208,215 ----
      PAssert(domain[1].first() == 0);
      PAssert(domain[2].first() == 0);
      PAssert(domain[3].first() == 0);
+     LHS localLHS(lhs);
+     RHS localRHS(rhs);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      int e2 = domain[2].length();
*************** struct KernelEvaluator<InlineKernelTag>
*** 210,216 ****
        for (int i2=0; i2<e2; ++i2)
  	for (int i1=0; i1<e1; ++i1)
  	  for (int i0=0; i0<e0; ++i0)
! 	    op(lhs(i0,i1,i2,i3),rhs.read(i0,i1,i2,i3));
    }
  
    template<class LHS,class Op,class RHS,class Domain>
--- 218,224 ----
        for (int i2=0; i2<e2; ++i2)
  	for (int i1=0; i1<e1; ++i1)
  	  for (int i0=0; i0<e0; ++i0)
! 	    op(localLHS(i0,i1,i2,i3),localRHS.read(i0,i1,i2,i3));
    }
  
    template<class LHS,class Op,class RHS,class Domain>
*************** struct KernelEvaluator<InlineKernelTag>
*** 223,228 ****
--- 231,238 ----
      PAssert(domain[2].first() == 0);
      PAssert(domain[3].first() == 0);
      PAssert(domain[4].first() == 0);
+     LHS localLHS(lhs);
+     RHS localRHS(rhs);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      int e2 = domain[2].length();
*************** struct KernelEvaluator<InlineKernelTag>
*** 233,239 ****
  	for (int i2=0; i2<e2; ++i2)
  	  for (int i1=0; i1<e1; ++i1)
  	    for (int i0=0; i0<e0; ++i0)
! 	      op(lhs(i0,i1,i2,i3,i4),rhs.read(i0,i1,i2,i3,i4));
    }
  
    template<class LHS,class Op,class RHS,class Domain>
--- 243,249 ----
  	for (int i2=0; i2<e2; ++i2)
  	  for (int i1=0; i1<e1; ++i1)
  	    for (int i0=0; i0<e0; ++i0)
! 	      op(localLHS(i0,i1,i2,i3,i4),localRHS.read(i0,i1,i2,i3,i4));
    }
  
    template<class LHS,class Op,class RHS,class Domain>
*************** struct KernelEvaluator<InlineKernelTag>
*** 247,252 ****
--- 257,264 ----
      PAssert(domain[3].first() == 0);
      PAssert(domain[4].first() == 0);
      PAssert(domain[5].first() == 0);
+     LHS localLHS(lhs);
+     RHS localRHS(rhs);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      int e2 = domain[2].length();
*************** struct KernelEvaluator<InlineKernelTag>
*** 259,266 ****
  	  for (int i2=0; i2<e2; ++i2)
  	    for (int i1=0; i1<e1; ++i1)
  	      for (int i0=0; i0<e0; ++i0)
! 		op(lhs(i0,i1,i2,i3,i4,i5),
! 		   rhs.read(i0,i1,i2,i3,i4,i5));
    }
  
    template<class LHS,class Op,class RHS,class Domain>
--- 271,278 ----
  	  for (int i2=0; i2<e2; ++i2)
  	    for (int i1=0; i1<e1; ++i1)
  	      for (int i0=0; i0<e0; ++i0)
! 		op(localLHS(i0,i1,i2,i3,i4,i5),
! 		   localRHS.read(i0,i1,i2,i3,i4,i5));
    }
  
    template<class LHS,class Op,class RHS,class Domain>
*************** struct KernelEvaluator<InlineKernelTag>
*** 275,280 ****
--- 287,294 ----
      PAssert(domain[4].first() == 0);
      PAssert(domain[5].first() == 0);
      PAssert(domain[6].first() == 0);
+     LHS localLHS(lhs);
+     RHS localRHS(rhs);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      int e2 = domain[2].length();
*************** struct KernelEvaluator<InlineKernelTag>
*** 289,296 ****
  	    for (int i2=0; i2<e2; ++i2)
  	      for (int i1=0; i1<e1; ++i1)
  		for (int i0=0; i0<e0; ++i0)
! 		  op(lhs(i0,i1,i2,i3,i4,i5,i6),
! 		     rhs.read(i0,i1,i2,i3,i4,i5,i6));
    }
  
  private:
--- 303,310 ----
  	    for (int i2=0; i2<e2; ++i2)
  	      for (int i1=0; i1<e1; ++i1)
  		for (int i0=0; i0<e0; ++i0)
! 		  op(localLHS(i0,i1,i2,i3,i4,i5,i6),
! 		     localRHS.read(i0,i1,i2,i3,i4,i5,i6));
    }
  
  private:
Index: ReductionEvaluator.h
===================================================================
RCS file: /home/pooma/Repository/r2/src/Evaluator/ReductionEvaluator.h,v
retrieving revision 1.5
diff -c -p -r1.5 ReductionEvaluator.h
*** ReductionEvaluator.h	2001/09/13 19:27:58	1.5
--- ReductionEvaluator.h	2001/11/03 00:15:14
*************** struct ReductionEvaluator<InlineKernelTa
*** 124,134 ****
    {
      CTAssert(Domain::unitStride);
      PAssert(domain[0].first() == 0);
      int e0 = domain[0].length();
  
!     ret = e.read(0);
      for (int i0 = 1; i0 < e0; ++i0)
!       op(ret, e.read(i0));
    }
  
    template<class T, class Op, class Expr, class Domain>
--- 124,137 ----
    {
      CTAssert(Domain::unitStride);
      PAssert(domain[0].first() == 0);
+     Expr localExpr(e);
      int e0 = domain[0].length();
  
!     T answer(localExpr.read(0));
      for (int i0 = 1; i0 < e0; ++i0)
!       op(answer, localExpr.read(i0));
! 
!     ret = answer;
    }
  
    template<class T, class Op, class Expr, class Domain>
*************** struct ReductionEvaluator<InlineKernelTa
*** 138,150 ****
      CTAssert(Domain::unitStride);
      PAssert(domain[0].first() == 0);
      PAssert(domain[1].first() == 0);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
  
      int i00;
      bool firstLoop = true;
      
!     ret = e.read(0, 0);
      for (int i1 = 0; i1 < e1; ++i1)
        {
          if (firstLoop)
--- 141,154 ----
      CTAssert(Domain::unitStride);
      PAssert(domain[0].first() == 0);
      PAssert(domain[1].first() == 0);
+     Expr localExpr(e);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
  
      int i00;
      bool firstLoop = true;
      
!     T answer(localExpr.read(0, 0));
      for (int i1 = 0; i1 < e1; ++i1)
        {
          if (firstLoop)
*************** struct ReductionEvaluator<InlineKernelTa
*** 155,162 ****
          else
            i00 = 0;
          for (int i0 = i00; i0 < e0; ++i0)
!           op(ret, e.read(i0, i1));
        }
    }
    
    template<class T, class Op, class Expr, class Domain>
--- 159,168 ----
          else
            i00 = 0;
          for (int i0 = i00; i0 < e0; ++i0)
!           op(answer, localExpr.read(i0, i1));
        }
+ 
+     ret = answer;
    }
    
    template<class T, class Op, class Expr, class Domain>
*************** struct ReductionEvaluator<InlineKernelTa
*** 167,172 ****
--- 173,179 ----
      PAssert(domain[0].first() == 0);
      PAssert(domain[1].first() == 0);
      PAssert(domain[2].first() == 0);
+     Expr localExpr(e);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      int e2 = domain[2].length();
*************** struct ReductionEvaluator<InlineKernelTa
*** 174,180 ****
      int i00;
      bool firstLoop = true;
      
!     ret = e.read(0, 0, 0);
      for (int i2 = 0; i2 < e2; ++i2)
        for (int i1 = 0; i1 < e1; ++i1)
          {
--- 181,187 ----
      int i00;
      bool firstLoop = true;
      
!     T answer(localExpr.read(0, 0, 0));
      for (int i2 = 0; i2 < e2; ++i2)
        for (int i1 = 0; i1 < e1; ++i1)
          {
*************** struct ReductionEvaluator<InlineKernelTa
*** 186,193 ****
            else
              i00 = 0;
            for (int i0 = i00; i0 < e0; ++i0)
!             op(ret, e.read(i0, i1, i2));
          }
    }
  
    template<class T, class Op, class Expr, class Domain>
--- 193,202 ----
            else
              i00 = 0;
            for (int i0 = i00; i0 < e0; ++i0)
!             op(answer, localExpr.read(i0, i1, i2));
          }
+     
+     ret = answer;
    }
  
    template<class T, class Op, class Expr, class Domain>
*************** struct ReductionEvaluator<InlineKernelTa
*** 199,204 ****
--- 208,214 ----
      PAssert(domain[1].first() == 0);
      PAssert(domain[2].first() == 0);
      PAssert(domain[3].first() == 0);
+     Expr localExpr(e);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      int e2 = domain[2].length();
*************** struct ReductionEvaluator<InlineKernelTa
*** 207,213 ****
      int i00;
      bool firstLoop = true;
      
!     ret = e.read(0, 0, 0, 0);
      for (int i3 = 0; i3 < e3; ++i3)
        for (int i2 = 0; i2 < e2; ++i2)
          for (int i1 = 0; i1 < e1; ++i1)
--- 217,223 ----
      int i00;
      bool firstLoop = true;
      
!     T answer(localExpr.read(0, 0, 0, 0));
      for (int i3 = 0; i3 < e3; ++i3)
        for (int i2 = 0; i2 < e2; ++i2)
          for (int i1 = 0; i1 < e1; ++i1)
*************** struct ReductionEvaluator<InlineKernelTa
*** 220,227 ****
              else
                i00 = 0;
              for (int i0 = i00; i0 < e0; ++i0)
!               op(ret, e.read(i0, i1, i2, i3));
            }
    }
  
    template<class T, class Op, class Expr, class Domain>
--- 230,239 ----
              else
                i00 = 0;
              for (int i0 = i00; i0 < e0; ++i0)
!               op(answer, localExpr.read(i0, i1, i2, i3));
            }
+ 
+     ret = answer;
    }
  
    template<class T, class Op, class Expr, class Domain>
*************** struct ReductionEvaluator<InlineKernelTa
*** 234,239 ****
--- 246,252 ----
      PAssert(domain[2].first() == 0);
      PAssert(domain[3].first() == 0);
      PAssert(domain[4].first() == 0);
+     Expr localExpr(e);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      int e2 = domain[2].length();
*************** struct ReductionEvaluator<InlineKernelTa
*** 243,249 ****
      int i00;
      bool firstLoop = true;
      
!     ret = e.read(0, 0, 0, 0, 0);
      for (int i4 = 0; i4 < e4; ++i4)
        for (int i3 = 0; i3 < e3; ++i3)
          for (int i2 = 0; i2 < e2; ++i2)
--- 256,262 ----
      int i00;
      bool firstLoop = true;
      
!     T answer(localExpr.read(0, 0, 0, 0, 0));
      for (int i4 = 0; i4 < e4; ++i4)
        for (int i3 = 0; i3 < e3; ++i3)
          for (int i2 = 0; i2 < e2; ++i2)
*************** struct ReductionEvaluator<InlineKernelTa
*** 257,264 ****
                else
                  i00 = 0;
                for (int i0 = i00; i0 < e0; ++i0)
!                 op(ret, e.read(i0, i1, i2, i3, i4));
              }
    }
  
    template<class T, class Op, class Expr, class Domain>
--- 270,279 ----
                else
                  i00 = 0;
                for (int i0 = i00; i0 < e0; ++i0)
!                 op(answer, localExpr.read(i0, i1, i2, i3, i4));
              }
+ 
+     ret = answer;
    }
  
    template<class T, class Op, class Expr, class Domain>
*************** struct ReductionEvaluator<InlineKernelTa
*** 272,277 ****
--- 287,293 ----
      PAssert(domain[3].first() == 0);
      PAssert(domain[4].first() == 0);
      PAssert(domain[5].first() == 0);
+     Expr localExpr(e);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      int e2 = domain[2].length();
*************** struct ReductionEvaluator<InlineKernelTa
*** 282,288 ****
      int i00;
      bool firstLoop = true;
      
!     ret = e.read(0, 0, 0, 0, 0, 0);
      for (int i5 = 0; i5 < e5; ++i5)
        for (int i4 = 0; i4 < e4; ++i4)
          for (int i3 = 0; i3 < e3; ++i3)
--- 298,304 ----
      int i00;
      bool firstLoop = true;
      
!     T answer(localExpr.read(0, 0, 0, 0, 0, 0));
      for (int i5 = 0; i5 < e5; ++i5)
        for (int i4 = 0; i4 < e4; ++i4)
          for (int i3 = 0; i3 < e3; ++i3)
*************** struct ReductionEvaluator<InlineKernelTa
*** 297,304 ****
                  else
                    i00 = 0;
                  for (int i0 = i00; i0 < e0; ++i0)
!                   op(ret, e.read(i0, i1, i2, i3, i4, i5));
                }
    }
  
    template<class T, class Op, class Expr, class Domain>
--- 313,322 ----
                  else
                    i00 = 0;
                  for (int i0 = i00; i0 < e0; ++i0)
!                   op(answer, localExpr.read(i0, i1, i2, i3, i4, i5));
                }
+ 
+     ret = answer;
    }
  
    template<class T, class Op, class Expr, class Domain>
*************** struct ReductionEvaluator<InlineKernelTa
*** 313,318 ****
--- 331,337 ----
      PAssert(domain[4].first() == 0);
      PAssert(domain[5].first() == 0);
      PAssert(domain[6].first() == 0);
+     Expr localExpr(e);
      int e0 = domain[0].length();
      int e1 = domain[1].length();
      int e2 = domain[2].length();
*************** struct ReductionEvaluator<InlineKernelTa
*** 324,330 ****
      int i00;
      bool firstLoop = true;
      
!     ret = e.read(0, 0, 0, 0, 0, 0, 0);
      for (int i6 = 0; i6 < e6; ++i6)
        for (int i5 = 0; i5 < e5; ++i5)
          for (int i4 = 0; i4 < e4; ++i4)
--- 343,349 ----
      int i00;
      bool firstLoop = true;
      
!     T answer(localExpr.read(0, 0, 0, 0, 0, 0, 0));
      for (int i6 = 0; i6 < e6; ++i6)
        for (int i5 = 0; i5 < e5; ++i5)
          for (int i4 = 0; i4 < e4; ++i4)
*************** struct ReductionEvaluator<InlineKernelTa
*** 340,347 ****
                    else
                      i00 = 0;
                    for (int i0 = i00; i0 < e0; ++i0)
!                     op(ret, e.read(i0, i1, i2, i3, i4, i5, i6));
                  }
    }
  };
  
--- 359,368 ----
                    else
                      i00 = 0;
                    for (int i0 = i00; i0 < e0; ++i0)
!                     op(answer, localExpr.read(i0, i1, i2, i3, i4, i5, i6));
                  }
+ 
+     ret = answer;
    }
  };