vg
tools for working with variation graphs
Classes | Typedefs | Functions
vg::io Namespace Reference

Classes

class  BlockedGzipInputStream
 
class  BlockedGzipOutputStream
 
class  fdinbuf
 
class  fdistream
 
class  fdostream
 
class  fdoutbuf
 
struct  hFILE_cppstream
 
class  JSONStreamHelper
 
class  MessageEmitter
 
class  MessageIterator
 
class  ProtobufEmitter
 
class  ProtobufIterator
 
class  Registry
 
class  StreamMultiplexer
 
class  VPKG
 

Typedefs

using message_consumer_function_t = function< void(const string &)>
 This is the type of a function that can be fed a series of messages. More...
 
using message_sender_function_t = function< void(const message_consumer_function_t &)>
 This is the type of a function that can be given a message consumer to feed messages to. More...
 
using load_function_t = function< void *(const message_sender_function_t &)>
 This is the type of a function that can allocate and load an object of unspecified type from a message source. More...
 
using save_function_t = function< void(const void *, const message_consumer_function_t &)>
 This is the type of a function that can serialize an object of unspecified type to a message consumer. More...
 
using bare_load_function_t = function< void *(istream &)>
 This is the type of a function that can load an object of unspecified type from a bare input stream. More...
 
using bare_save_function_t = function< void(const void *, ostream &)>
 This is the type of a function that can save an object of unspecified type to a bare output stream. More...
 

Functions

void load_proto_to_graph (vg::MutablePathMutableHandleGraph *destination, const vg::io::message_sender_function_t &for_each_message)
 
void load_proto_to_graph (vg::MutablePathMutableHandleGraph *destination, const function< void(const function< void(Graph &)> &)> &chunk_sender)
 
bool register_libvg_io ()
 
void register_loader_saver_distance_index ()
 
void register_loader_saver_gbwt ()
 
void register_loader_saver_gbwtgraph ()
 
void register_loader_saver_gcsa ()
 
void register_loader_saver_hash_graph ()
 
void register_loader_saver_lcp ()
 
void register_loader_saver_minimizer ()
 
void register_loader_saver_odgi ()
 
void register_loader_saver_packed_graph ()
 
void register_loader_saver_snarl_manager ()
 
void register_loader_saver_vg ()
 
void register_loader_saver_xg ()
 
void save_handle_graph (HandleGraph *graph, ostream &os)
 
void save_handle_graph (HandleGraph *graph, const string &dest_path)
 
bool valid_output_format (const string &fmt_string)
 
template<class T >
unique_ptr< T > new_output_graph (const string &fmt_string)
 
void mergeGraphs (Graph &graph, const Graph &part)
 
Graph inputStream (const string &filename)
 
void outputStream (const Graph &g)
 
hFILE * hfile_wrap (std::istream &input)
 Wrap a C++ input stream as an hFILE* that can be read by BGZF. More...
 
hFILE * hfile_wrap (std::ostream &output)
 Wrap a C++ output stream as an hFILE* that can be written by BGZF. More...
 
auto wrap_bare_loader (function< void *(istream &)> istream_loader) -> load_function_t
 
void with_function_calling_stream (const message_consumer_function_t &emit_message, const function< void(ostream &)> &use_stream)
 
auto wrap_bare_saver (function< void(const void *, ostream &)> ostream_saver) -> save_function_t
 
void finish (std::ostream &out, bool compressed)
 
template<typename Item >
std::function< void(const Item &)> emit_to (ostream &out)
 
template<typename Item >
auto emit_to (ostream &out) -> std::function< void(const Item &)>
 
template<typename T >
bool write (std::ostream &out, size_t count, const std::function< T &(size_t)> &lambda, bool compressed=true)
 
template<typename T >
bool write (std::ostream &out, size_t count, const std::function< T(size_t)> &lambda, bool compressed=true)
 
template<typename T >
bool write_buffered (std::ostream &out, std::vector< T > &buffer, size_t buffer_limit, bool compressed=true)
 
template<typename T >
void write_to_file (const T &item, const string &filename)
 Write a single message to a file. More...
 
template<typename T >
void for_each (std::istream &in, const std::function< void(int64_t, T &)> &lambda)
 
template<typename T >
void for_each (std::istream &in, const std::function< void(T &)> &lambda)
 
template<typename T >
void for_each_parallel_impl (std::istream &in, const std::function< void(T &, T &)> &lambda2, const std::function< void(T &)> &lambda1, const std::function< bool(void)> &single_threaded_until_true, size_t batch_size)
 
template<typename T >
void for_each_interleaved_pair_parallel (std::istream &in, const std::function< void(T &, T &)> &lambda2, size_t batch_size=256)
 
template<typename T >
void for_each_interleaved_pair_parallel_after_wait (std::istream &in, const std::function< void(T &, T &)> &lambda2, const std::function< bool(void)> &single_threaded_until_true, size_t batch_size=256)
 
template<typename T >
void for_each_parallel (std::istream &in, const std::function< void(T &)> &lambda1, size_t batch_size=256)
 

Typedef Documentation

◆ bare_load_function_t

using vg::io::bare_load_function_t = typedef function<void*(istream&)>

This is the type of a function that can load an object of unspecified type from a bare input stream.

◆ bare_save_function_t

using vg::io::bare_save_function_t = typedef function<void(const void*, ostream&)>

This is the type of a function that can save an object of unspecified type to a bare output stream.

◆ load_function_t

using vg::io::load_function_t = typedef function<void*(const message_sender_function_t&)>

This is the type of a function that can allocate and load an object of unspecified type from a message source.

◆ message_consumer_function_t

using vg::io::message_consumer_function_t = typedef function<void(const string&)>

This is the type of a function that can be fed a series of messages.

◆ message_sender_function_t

using vg::io::message_sender_function_t = typedef function<void(const message_consumer_function_t&)>

This is the type of a function that can be given a message consumer to feed messages to.

◆ save_function_t

using vg::io::save_function_t = typedef function<void(const void*, const message_consumer_function_t&)>

This is the type of a function that can serialize an object of unspecified type to a message consumer.

Function Documentation

◆ emit_to() [1/2]

template<typename Item >
std::function<void(const Item&)> vg::io::emit_to ( ostream &  out)

Produce an std::function that can be invoked with Protobuf objects and save them to the given stream. Easy way to get a dumping callback to feed to something that wants a callback. The passed stream must outlive the resulting function.

◆ emit_to() [2/2]

template<typename Item >
auto vg::io::emit_to ( ostream &  out)

Produce an std::function that can be invoked with Protobuf objects and save them to the given stream. Easy way to get a dumping callback to feed to something that wants a callback. The passed stream must outlive the resulting function.

◆ finish()

void vg::io::finish ( std::ostream &  out,
bool  compressed = true 
)

Write the EOF marker to the given stream, so that readers won't complain that it might be truncated when they read it in. Internal EOF markers MAY exist, but a file SHOULD have exactly one EOF marker at its end. Needs to know if the output stream is compressed or not. Note that uncompressed streams don't actually have nonempty EOF markers.

◆ for_each() [1/2]

template<typename T >
void vg::io::for_each ( std::istream &  in,
const std::function< void(int64_t, T &)> &  lambda 
)

◆ for_each() [2/2]

template<typename T >
void vg::io::for_each ( std::istream &  in,
const std::function< void(T &)> &  lambda 
)

◆ for_each_interleaved_pair_parallel()

template<typename T >
void vg::io::for_each_interleaved_pair_parallel ( std::istream &  in,
const std::function< void(T &, T &)> &  lambda2,
size_t  batch_size = 256 
)

◆ for_each_interleaved_pair_parallel_after_wait()

template<typename T >
void vg::io::for_each_interleaved_pair_parallel_after_wait ( std::istream &  in,
const std::function< void(T &, T &)> &  lambda2,
const std::function< bool(void)> &  single_threaded_until_true,
size_t  batch_size = 256 
)

◆ for_each_parallel()

template<typename T >
void vg::io::for_each_parallel ( std::istream &  in,
const std::function< void(T &)> &  lambda1,
size_t  batch_size = 256 
)

◆ for_each_parallel_impl()

template<typename T >
void vg::io::for_each_parallel_impl ( std::istream &  in,
const std::function< void(T &, T &)> &  lambda2,
const std::function< void(T &)> &  lambda1,
const std::function< bool(void)> &  single_threaded_until_true,
size_t  batch_size 
)

◆ hfile_wrap() [1/2]

hFILE * vg::io::hfile_wrap ( std::istream &  input)

Wrap a C++ input stream as an hFILE* that can be read by BGZF.

Make the base struct, making sure it knows how big we are

◆ hfile_wrap() [2/2]

hFILE * vg::io::hfile_wrap ( std::ostream &  output)

Wrap a C++ output stream as an hFILE* that can be written by BGZF.

Make the base struct, making sure it knows how big we are

◆ inputStream()

Graph vg::io::inputStream ( const string &  filename)

◆ load_proto_to_graph() [1/2]

void vg::io::load_proto_to_graph ( vg::MutablePathMutableHandleGraph destination,
const function< void(const function< void(Graph &)> &)> &  chunk_sender 
)

Call the given function with a callback which it can call with a series of Protobuf Graph objects, possibly in multiple threads. The Protobuf Graph objects may have dangling edges.

Resolves all the dangling edges and writes all the graph data into the given MutablePathMutableHandleGraph, with the destination graph being protected from concurrent modification.

◆ load_proto_to_graph() [2/2]

void vg::io::load_proto_to_graph ( vg::MutablePathMutableHandleGraph destination,
const vg::io::message_sender_function_t for_each_message 
)

Read all string messages supplied by the given message sender as Protobuf Graph objects, and create the specified graph in the destination graph.

Paths need to be cached until the end for ranks to be respected.

◆ mergeGraphs()

void vg::io::mergeGraphs ( Graph graph,
const Graph part 
)

◆ new_output_graph()

template<class T >
unique_ptr<T> vg::io::new_output_graph ( const string &  fmt_string)

◆ outputStream()

void vg::io::outputStream ( const Graph g)

◆ register_libvg_io()

bool vg::io::register_libvg_io ( )

Register libvg types with libvgio. Must be called by library users before doing IO. Does not magically statically call itself. TODO: work out a way it can. Returns true on success.

◆ register_loader_saver_distance_index()

void vg::io::register_loader_saver_distance_index ( )

◆ register_loader_saver_gbwt()

void vg::io::register_loader_saver_gbwt ( )

◆ register_loader_saver_gbwtgraph()

void vg::io::register_loader_saver_gbwtgraph ( )

◆ register_loader_saver_gcsa()

void vg::io::register_loader_saver_gcsa ( )

◆ register_loader_saver_hash_graph()

void vg::io::register_loader_saver_hash_graph ( )

◆ register_loader_saver_lcp()

void vg::io::register_loader_saver_lcp ( )

◆ register_loader_saver_minimizer()

void vg::io::register_loader_saver_minimizer ( )

◆ register_loader_saver_odgi()

void vg::io::register_loader_saver_odgi ( )

◆ register_loader_saver_packed_graph()

void vg::io::register_loader_saver_packed_graph ( )

◆ register_loader_saver_snarl_manager()

void vg::io::register_loader_saver_snarl_manager ( )

◆ register_loader_saver_vg()

void vg::io::register_loader_saver_vg ( )

◆ register_loader_saver_xg()

void vg::io::register_loader_saver_xg ( )

◆ save_handle_graph() [1/2]

void vg::io::save_handle_graph ( HandleGraph graph,
const string &  dest_path 
)
inline

◆ save_handle_graph() [2/2]

void vg::io::save_handle_graph ( HandleGraph graph,
ostream &  os 
)
inline

Save a handle graph using the VPKG::save() function. Todo: should this be somewhere else (ie in vgio with new types registered?)

◆ valid_output_format()

bool vg::io::valid_output_format ( const string &  fmt_string)
inline

◆ with_function_calling_stream()

void vg::io::with_function_calling_stream ( const message_consumer_function_t emit_message,
const function< void(ostream &)> &  use_stream 
)

This calls the given stream-using callback with a stream that, when written to, calls the given emit_message function. The emit_message function and the stream-using callback will run in different threads.

◆ wrap_bare_loader()

load_function_t vg::io::wrap_bare_loader ( bare_load_function_t  istream_loader)

We also have an adapter that takes a function from an istream& to a void* object, and runs that in a thread to adapt it to the message consuming shape of interface. It captures the wrapped function by value.

◆ wrap_bare_saver()

save_function_t vg::io::wrap_bare_saver ( function< void(const void *, ostream &)>  ostream_saver)

We have an adapter that takes a function of void* and ostream&, and adapts that to a message consumer destination.

◆ write() [1/2]

template<typename T >
bool vg::io::write ( std::ostream &  out,
size_t  count,
const std::function< T &(size_t)> &  lambda,
bool  compressed = true 
)

Write objects. count should be equal to the number of objects to write. count is written before the objects, but if it is 0, it is not written. To get the objects, calls lambda with the index of the object to retrieve. If not all objects are written, return false, otherwise true. Needs to know whether to BGZF-compress the output or not.

◆ write() [2/2]

template<typename T >
bool vg::io::write ( std::ostream &  out,
size_t  count,
const std::function< T(size_t)> &  lambda,
bool  compressed = true 
)

Write objects. count should be equal to the number of objects to write. count is written before the objects, but if it is 0, it is not written. To get the objects, calls lambda with the index of the object to retrieve. If not all objects are written, return false, otherwise true. This implementation takes a function that returns actual objects and not references. Needs to know whether to BGZF-compress the output or not.

◆ write_buffered()

template<typename T >
bool vg::io::write_buffered ( std::ostream &  out,
std::vector< T > &  buffer,
size_t  buffer_limit,
bool  compressed = true 
)

Start, continue, or finish a buffered stream of objects. If the length of the buffer is greater than the limit, writes the buffer out. Otherwise, leaves the objects in the buffer. Must be called with a buffer limit of 0 after all the objects have been produced, to flush the buffer. When called with a buffer limit of 0, automatically appends an EOF marker. Returns true unless an error occurs. Needs to know whether to BGZF-compress the output or not.

◆ write_to_file()

template<typename T >
void vg::io::write_to_file ( const T &  item,
const string &  filename 
)

Write a single message to a file.