Add SharedDataset (#13800)

Summary:
This PR adds a `SharedDataset` to the C++ frontend data API, which allows wrapping a shared_ptr to a dataset into a class that conforms to the `Dataset` interface (with `get_batch`). This enables use cases where a custom dataset is (1) thread-safe and (2) expensive to copy. All workers will reference a single instance of this dataset. No additional copies are incurred.

jaliyae apaszke
Pull Request resolved: https://github.com/pytorch/pytorch/pull/13800

Differential Revision: D13075610

Pulled By: goldsborough

fbshipit-source-id: 4ffdfd7959d49b042c0e254110085f62a0bfeb6c
This commit is contained in:
Peter Goldsborough
2018-11-16 13:01:25 -08:00
committed by Facebook Github Bot
parent 96e5d23bad
commit fb6535ec70
3 changed files with 128 additions and 0 deletions
+49
View File
@@ -15,9 +15,11 @@
#include <iostream>
#include <iterator>
#include <limits>
#include <mutex>
#include <stdexcept>
#include <string>
#include <thread>
#include <unordered_set>
#include <vector>
using namespace torch::data; // NOLINT
@@ -539,6 +541,53 @@ TEST(DataTest, DataShuttlePopResultTimesOut) {
ASSERT_THROWS_WITH(shuttle.pop_result(10 * kMillisecond), "Timeout");
}
struct UncopyableDataset : datasets::Dataset<UncopyableDataset, int> {
UncopyableDataset(const std::string& /* unused */)
: mutex(torch::make_unique<std::mutex>()) {}
UncopyableDataset(UncopyableDataset&&) = default;
UncopyableDataset& operator=(UncopyableDataset&&) = default;
UncopyableDataset(const UncopyableDataset&) = delete;
UncopyableDataset& operator=(const UncopyableDataset&) = delete;
int get(size_t index) override {
{
std::lock_guard<std::mutex> guard(*mutex);
thread_ids.insert(std::this_thread::get_id());
}
return 1 + index;
}
torch::optional<size_t> size() const override {
return 100;
}
std::unique_ptr<std::mutex> mutex;
std::unordered_set<std::thread::id> thread_ids;
};
TEST(DataTest, SharedBatchDatasetReallyIsShared) {
auto shared_dataset =
torch::data::datasets::make_shared_dataset<UncopyableDataset>(
"uncopyable");
auto data_loader = torch::data::make_data_loader(
shared_dataset, torch::data::DataLoaderOptions().workers(3));
for (auto batch : *data_loader) {
/* exhaust */
}
ASSERT_EQ(shared_dataset->thread_ids.size(), 3);
}
TEST(DataTest, SharedBatchDatasetDoesNotIncurCopyWhenPassedDatasetObject) {
// This will not compile if a copy is made.
auto shared_dataset =
torch::data::datasets::make_shared_dataset<UncopyableDataset>(
UncopyableDataset("uncopyable"));
ASSERT_EQ(shared_dataset.size().value(), 100);
}
struct TestIndex : public torch::data::samplers::CustomBatchRequest {
explicit TestIndex(size_t offset, std::vector<size_t> index)
: offset(offset), index(std::move(index)) {}