mirror of
https://github.com/pytorch/pytorch.git
synced 2026-11-20 09:04:25 +08:00
Add SharedDataset (#13800)
Summary: This PR adds a `SharedDataset` to the C++ frontend data API, which allows wrapping a shared_ptr to a dataset into a class that conforms to the `Dataset` interface (with `get_batch`). This enables use cases where a custom dataset is (1) thread-safe and (2) expensive to copy. All workers will reference a single instance of this dataset. No additional copies are incurred. jaliyae apaszke Pull Request resolved: https://github.com/pytorch/pytorch/pull/13800 Differential Revision: D13075610 Pulled By: goldsborough fbshipit-source-id: 4ffdfd7959d49b042c0e254110085f62a0bfeb6c
This commit is contained in:
committed by
Facebook Github Bot
parent
96e5d23bad
commit
fb6535ec70
@@ -15,9 +15,11 @@
|
||||
#include <iostream>
|
||||
#include <iterator>
|
||||
#include <limits>
|
||||
#include <mutex>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
using namespace torch::data; // NOLINT
|
||||
@@ -539,6 +541,53 @@ TEST(DataTest, DataShuttlePopResultTimesOut) {
|
||||
ASSERT_THROWS_WITH(shuttle.pop_result(10 * kMillisecond), "Timeout");
|
||||
}
|
||||
|
||||
struct UncopyableDataset : datasets::Dataset<UncopyableDataset, int> {
|
||||
UncopyableDataset(const std::string& /* unused */)
|
||||
: mutex(torch::make_unique<std::mutex>()) {}
|
||||
|
||||
UncopyableDataset(UncopyableDataset&&) = default;
|
||||
UncopyableDataset& operator=(UncopyableDataset&&) = default;
|
||||
|
||||
UncopyableDataset(const UncopyableDataset&) = delete;
|
||||
UncopyableDataset& operator=(const UncopyableDataset&) = delete;
|
||||
|
||||
int get(size_t index) override {
|
||||
{
|
||||
std::lock_guard<std::mutex> guard(*mutex);
|
||||
thread_ids.insert(std::this_thread::get_id());
|
||||
}
|
||||
return 1 + index;
|
||||
}
|
||||
torch::optional<size_t> size() const override {
|
||||
return 100;
|
||||
}
|
||||
|
||||
std::unique_ptr<std::mutex> mutex;
|
||||
std::unordered_set<std::thread::id> thread_ids;
|
||||
};
|
||||
|
||||
TEST(DataTest, SharedBatchDatasetReallyIsShared) {
|
||||
auto shared_dataset =
|
||||
torch::data::datasets::make_shared_dataset<UncopyableDataset>(
|
||||
"uncopyable");
|
||||
auto data_loader = torch::data::make_data_loader(
|
||||
shared_dataset, torch::data::DataLoaderOptions().workers(3));
|
||||
|
||||
for (auto batch : *data_loader) {
|
||||
/* exhaust */
|
||||
}
|
||||
|
||||
ASSERT_EQ(shared_dataset->thread_ids.size(), 3);
|
||||
}
|
||||
|
||||
TEST(DataTest, SharedBatchDatasetDoesNotIncurCopyWhenPassedDatasetObject) {
|
||||
// This will not compile if a copy is made.
|
||||
auto shared_dataset =
|
||||
torch::data::datasets::make_shared_dataset<UncopyableDataset>(
|
||||
UncopyableDataset("uncopyable"));
|
||||
ASSERT_EQ(shared_dataset.size().value(), 100);
|
||||
}
|
||||
|
||||
struct TestIndex : public torch::data::samplers::CustomBatchRequest {
|
||||
explicit TestIndex(size_t offset, std::vector<size_t> index)
|
||||
: offset(offset), index(std::move(index)) {}
|
||||
|
||||
Reference in New Issue
Block a user