mirror of
https://github.com/pytorch/pytorch.git
synced 2026-11-20 09:04:25 +08:00
Sampler serialization and deserialization (#12999)
Summary: Implements serialization and deserialization for samplers in the C++ frontend dataloader. apaszke Pull Request resolved: https://github.com/pytorch/pytorch/pull/12999 Differential Revision: D10859676 Pulled By: goldsborough fbshipit-source-id: cd132100fd35323e5a3df33e314511750806f48d
This commit is contained in:
committed by
Facebook Github Bot
parent
9f9f06c937
commit
c21471c77f
@@ -2,6 +2,7 @@
|
||||
|
||||
#include <torch/data.h>
|
||||
#include <torch/data/detail/sequencers.h>
|
||||
#include <torch/serialize.h>
|
||||
#include <torch/tensor.h>
|
||||
|
||||
#include <test/cpp/api/support.h>
|
||||
@@ -73,6 +74,9 @@ struct BatchSizeSampler : samplers::Sampler {
|
||||
c10::optional<std::vector<size_t>> next(size_t batch_size) override {
|
||||
return {{batch_size}};
|
||||
}
|
||||
|
||||
void save(torch::serialize::OutputArchive& archive) const override {}
|
||||
void load(torch::serialize::InputArchive& archive) override {}
|
||||
};
|
||||
|
||||
TEST(DataTest, InfiniteStreamDataset) {
|
||||
@@ -102,7 +106,6 @@ TEST(DataTest, InfiniteStreamDataset) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(DataTest, NoSequencerIsIdentity) {
|
||||
using namespace torch::data::detail::sequencers; // NOLINT
|
||||
NoSequencer<int> no_sequencer;
|
||||
@@ -212,6 +215,31 @@ TEST(DataTest, SequentialSamplerResetsWell) {
|
||||
ASSERT_FALSE(sampler.next(2).has_value());
|
||||
}
|
||||
|
||||
TEST(DataTest, CanSaveAndLoadSequentialSampler) {
|
||||
{
|
||||
samplers::SequentialSampler a(10);
|
||||
ASSERT_EQ(a.index(), 0);
|
||||
std::stringstream stream;
|
||||
torch::save(a, stream);
|
||||
|
||||
samplers::SequentialSampler b(10);
|
||||
torch::load(b, stream);
|
||||
ASSERT_EQ(b.index(), 0);
|
||||
}
|
||||
{
|
||||
samplers::SequentialSampler a(10);
|
||||
a.next(3);
|
||||
a.next(4);
|
||||
ASSERT_EQ(a.index(), 7);
|
||||
std::stringstream stream;
|
||||
torch::save(a, stream);
|
||||
|
||||
samplers::SequentialSampler b(10);
|
||||
torch::load(b, stream);
|
||||
ASSERT_EQ(b.index(), 7);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(DataTest, RandomSamplerReturnsIndicesInCorrectRange) {
|
||||
samplers::RandomSampler sampler(10);
|
||||
|
||||
@@ -227,6 +255,7 @@ TEST(DataTest, RandomSamplerReturnsIndicesInCorrectRange) {
|
||||
ASSERT_LT(i, 10);
|
||||
}
|
||||
|
||||
|
||||
indices = sampler.next(2).value();
|
||||
for (auto i : indices) {
|
||||
ASSERT_GE(i, 0);
|
||||
@@ -252,6 +281,36 @@ TEST(DataTest, RandomSamplerResetsWell) {
|
||||
ASSERT_FALSE(sampler.next(2).has_value());
|
||||
}
|
||||
|
||||
TEST(DataTest, SavingAndLoadingRandomSamplerYieldsSameSequence) {
|
||||
{
|
||||
samplers::RandomSampler a(10);
|
||||
|
||||
std::stringstream stream;
|
||||
torch::save(a, stream);
|
||||
|
||||
samplers::RandomSampler b(10);
|
||||
torch::load(b, stream);
|
||||
|
||||
ASSERT_EQ(a.next(10).value(), b.next(10).value());
|
||||
}
|
||||
{
|
||||
samplers::RandomSampler a(10);
|
||||
a.next(3);
|
||||
ASSERT_EQ(a.index(), 3);
|
||||
|
||||
std::stringstream stream;
|
||||
torch::save(a, stream);
|
||||
|
||||
samplers::RandomSampler b(10);
|
||||
torch::load(b, stream);
|
||||
ASSERT_EQ(b.index(), 3);
|
||||
|
||||
auto b_sequence = b.next(10).value();
|
||||
ASSERT_EQ(b_sequence.size(), 7);
|
||||
ASSERT_EQ(a.next(10).value(), b_sequence);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(DataTest, TensorDatasetConstructsFromSingleTensor) {
|
||||
datasets::TensorDataset dataset(torch::eye(5));
|
||||
ASSERT_TRUE(
|
||||
|
||||
Reference in New Issue
Block a user