casacore
Loading...
Searching...
No Matches
MultiHDF5.h
Go to the documentation of this file.
1//# MultiHDF5.h: Class to combine multiple files in a single HDF5 file
2//# Copyright (C) 2015
3//# Associated Universities, Inc. Washington DC, USA.
4//#
5//# This library is free software; you can redistribute it and/or modify it
6//# under the terms of the GNU Library General Public License as published by
7//# the Free Software Foundation; either version 2 of the License, or (at your
8//# option) any later version.
9//#
10//# This library is distributed in the hope that it will be useful, but WITHOUT
11//# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12//# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13//# License for more details.
14//#
15//# You should have received a copy of the GNU Library General Public License
16//# along with this library; if not, write to the Free Software Foundation,
17//# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18//#
19//# Correspondence concerning AIPS++ should be addressed as follows:
20//# Internet email: aips2-request@nrao.edu.
21//# Postal address: AIPS++ Project Office
22//# National Radio Astronomy Observatory
23//# 520 Edgemont Road
24//# Charlottesville, VA 22903-2475 USA
25//#
26//# $Id: RegularFileIO.h 20551 2009-03-25 00:11:33Z Malte.Marquarding $
27
28#ifndef CASA_MULTIHDF5_H
29#define CASA_MULTIHDF5_H
30
31//# Includes
32#include <casacore/casa/aips.h>
33#include <casacore/casa/IO/MultiFile.h>
34#include <casacore/casa/HDF5/HDF5File.h>
35
36
37namespace casacore { //# NAMESPACE CASACORE - BEGIN
38
39 // <summary>
40 // Class to combine multiple files in a single HDF5 file.
41 // </summary>
42
43 // <use visibility=export>
44
45 // <reviewed reviewer="" date="" tests="tMultiHDF5" demos="">
46 // </reviewed>
47
48 // <synopsis>
49 // This class is a container file holding multiple virtual files. It is
50 // primarily meant as a container file for the storage manager files of a
51 // table to reduce the number of files used (especially for Lustre) and to
52 // reduce the number of open files (especially when concatenating tables).
53 // <br>A secondary goal is offering the ability to use an IO buffer size
54 // that matches the file system well (large buffer size for e.g. ZFS).
55 //
56 // The SetupNewTable constructor has a StorageOption argument to define
57 // if a MultiFile has to be used and if so, the buffer size to use.
58 // It is also possible to specify that through aipsrc variables.
59 //
60 // A virtual file is spread over multiple (fixed size) data blocks in the
61 // MultiFile. A data block is never shared by multiple files.
62 // For each virtual file MultiFile keeps a MultiFileInfo object telling
63 // the file size and the blocks numbers used for the file. When flushing
64 // the MultiFile, this meta info is written into a header block and,
65 // if needed, continuation blocks. On open and resync, it is read back.
66 // <br>
67 //
68 // A virtual file is represented by an MFFileIO object, which is derived
69 // from ByteIO and as such part of the casacore IO framework. It makes it
70 // possible for applications to access a virtual file in the same way as
71 // a regular file.
72 //
73 // It is possible to delete a virtual file. Its blocks will be added to
74 // the free block list (which is also stored in the meta info).
75 // </synopsis>
76
77 // <example>
78 // In principle it is possible to use the MultiFile functions directly.
79 // However, in general it is much easier to use an MFFileIO object
80 // per virtual file as shown below.
81 // <srcblock>
82 // // Create a new MultiFile using a block size of 1 MB.
83 // MultiFile mfile("file.mf', ByteIO::New, 1048576);
84 // // Create a virtual file in it.
85 // MFFileIO mf1(mfile, "mf1", ByteIO::New);
86 // // Use it (for example) as the sink of AipsIO.
87 // AipsIO stream (&mf1);
88 // // Write values.
89 // stream << (Int)10;
90 // stream << True;
91 // // Seek to beginning of file and read data in.
92 // stream.setpos (0);
93 // Int vali;
94 // Bool valb;
95 // stream >> vali >> valb;
96 // </srcblock>
97 // </example>
98
99 // <todo>
100 // <li> write headers at alternating file positions (for robustness)
101 // <li> possibly write headers entirely at the end if larger than blocksize
102 // </todo>
103
104
106 {
107 public:
108 // Open or create a MultiHDF5 with the given name.
109 // Upon creation the block size can be given. If 0, it uses the block size
110 // of the file system the file is on.
112
113 // The destructor flushes and closes the file.
114 virtual ~MultiHDF5();
115
116 // Reopen the underlying file for read/write access.
117 // Nothing will be done if the file is writable already.
118 // Otherwise it will be reopened and an exception will be thrown
119 // if it is not possible to reopen it for read/write access.
120 virtual void reopenRW();
121
122 // Fsync the file (i.e., force the data to be physically written).
123 virtual void fsync();
124
125 private:
126 // Do the class-specific actions on adding a file.
127 virtual void doAddFile (MultiFileInfo&);
128 // Do the class-specific actions on deleting a file.
130 // Flush the file itself.
131 virtual void flushFile();
132 // Flush and close the file.
133 virtual void close();
134 // Write the header info.
135 virtual void writeHeader();
136 // Read the header info. If always==False, the info is only read if the
137 // header counter has changed.
138 virtual void readHeader (Bool always=True);
139 // Extend the virtual file to fit lastblk.
140 virtual void extend (MultiFileInfo& info, Int64 lastblk);
141 // Read a data block.
142 virtual void readBlock (MultiFileInfo& info, Int64 blknr,
143 void* buffer);
144 // Write a data block.
145 virtual void writeBlock (MultiFileInfo& info, Int64 blknr,
146 const void* buffer);
147
148 //# Data members
150 };
151
152
153} //# NAMESPACE CASACORE - END
154
155#endif
OpenOption
Define the possible ByteIO open options.
Definition ByteIO.h:65
Abstract base class to combine multiple files in a single one.
Int64 blockSize() const
Get the block size used.
const vector< MultiFileInfo > & info() const
Get the info object (for test purposes mainly).
virtual void fsync()
Fsync the file (i.e., force the data to be physically written).
virtual void readHeader(Bool always=True)
Read the header info.
virtual void writeHeader()
Write the header info.
virtual void doDeleteFile(MultiFileInfo &)
Do the class-specific actions on deleting a file.
virtual void close()
Flush and close the file.
virtual void writeBlock(MultiFileInfo &info, Int64 blknr, const void *buffer)
Write a data block.
virtual ~MultiHDF5()
The destructor flushes and closes the file.
virtual void readBlock(MultiFileInfo &info, Int64 blknr, void *buffer)
Read a data block.
virtual void extend(MultiFileInfo &info, Int64 lastblk)
Extend the virtual file to fit lastblk.
virtual void reopenRW()
Reopen the underlying file for read/write access.
virtual void doAddFile(MultiFileInfo &)
Do the class-specific actions on adding a file.
MultiHDF5(const String &name, ByteIO::OpenOption, Int blockSize=0)
Open or create a MultiHDF5 with the given name.
virtual void flushFile()
Flush the file itself.
String: the storage and methods of handling collections of characters.
Definition String.h:225
this file contains all the compiler specific defines
Definition mainpage.dox:28
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition aipsxtype.h:38
int Int
Definition aipstype.h:50
bool Bool
Define the standard types used by Casacore.
Definition aipstype.h:42
const Bool True
Definition aipstype.h:43
Helper class for MultiFileBase containing info per internal file.