diff --git a/daemon/main/Options.cpp b/daemon/main/Options.cpp index 0364dbe6..09803a31 100644 --- a/daemon/main/Options.cpp +++ b/daemon/main/Options.cpp @@ -345,6 +345,8 @@ void Options::InitDefaults() SetOption(RARRENAME.data(), "yes"); SetOption(HEALTHCHECK.data(), "none"); SetOption(DIRECTRENAME.data(), "no"); + SetOption(HARDLINKING.data(), "no"); + SetOption(HARDLINKINGIGNOREEXT.data(), ".zip, .7z, .rar, *.7z.###, *.r##"); SetOption(SCRIPTORDER.data(), ""); SetOption(EXTENSIONS.data(), ""); SetOption(DAEMONUSERNAME.data(), "root"); @@ -637,6 +639,8 @@ void Options::InitOptions() m_parRename = (bool)ParseEnumValue(PARRENAME.data(), BoolCount, BoolNames, BoolValues); m_rarRename = (bool)ParseEnumValue(RARRENAME.data(), BoolCount, BoolNames, BoolValues); m_directRename = (bool)ParseEnumValue(DIRECTRENAME.data(), BoolCount, BoolNames, BoolValues); + m_hardLinking = (bool)ParseEnumValue(HARDLINKING.data(), BoolCount, BoolNames, BoolValues); + m_hardLinkingIgnoreExt = GetOption(HARDLINKINGIGNOREEXT.data()); m_cursesNzbName = (bool)ParseEnumValue(CURSESNZBNAME.data(), BoolCount, BoolNames, BoolValues); m_cursesTime = (bool)ParseEnumValue(CURSESTIME.data(), BoolCount, BoolNames, BoolValues); m_cursesGroup = (bool)ParseEnumValue(CURSESGROUP.data(), BoolCount, BoolNames, BoolValues); diff --git a/daemon/main/Options.h b/daemon/main/Options.h index 10163a87..92d4bcab 100644 --- a/daemon/main/Options.h +++ b/daemon/main/Options.h @@ -106,6 +106,8 @@ class Options static constexpr std::string_view RARRENAME = "RarRename"; static constexpr std::string_view HEALTHCHECK = "HealthCheck"; static constexpr std::string_view DIRECTRENAME = "DirectRename"; + static constexpr std::string_view HARDLINKING = "HardLinking"; + static constexpr std::string_view HARDLINKINGIGNOREEXT = "HardLinkingIgnoreExt"; static constexpr std::string_view UMASK = "UMask"; static constexpr std::string_view UPDATEINTERVAL = "UpdateInterval"; static constexpr std::string_view CURSESNZBNAME = "CursesNzbName"; @@ -484,6 +486,8 @@ class Options int GetQuotaStartDay() const { return m_quotaStartDay; } int GetDailyQuota() const { return m_dailyQuota; } bool GetDirectRename() const { return m_directRename; } + bool GetHardLinking() const { return m_hardLinking; } + const char* GetHardLinkingIgnoreExt() const { return m_hardLinkingIgnoreExt; } bool GetReorderFiles() const { return m_reorderFiles; } EFileNaming GetFileNaming() const { return m_fileNaming; } int GetDownloadRate() const { return m_downloadRate; } @@ -619,6 +623,8 @@ class Options int m_parThreads = 0; bool m_rarRename = false; bool m_directRename = false; + bool m_hardLinking = false; + CString m_hardLinkingIgnoreExt; EHealthCheck m_healthCheck = hcNone; CString m_extensions; CString m_scriptOrder; diff --git a/daemon/nntp/ArticleWriter.cpp b/daemon/nntp/ArticleWriter.cpp index 7738417a..40b7be62 100644 --- a/daemon/nntp/ArticleWriter.cpp +++ b/daemon/nntp/ArticleWriter.cpp @@ -583,6 +583,20 @@ void ArticleWriter::CompleteFileParts() GuardedDownloadQueue guard = DownloadQueue::Guard(); m_fileInfo->SetCrc(crc); + if (m_fileInfo->IsHardLinked() && Util::EmptyStr(g_Options->GetInterDir())) + { + // No need to rename, remove hardlink of .out.tmp-file + if (FileSystem::DeleteFile(m_outputFilename.c_str())) + { + m_fileInfo->SetOutputFilename(nullptr); + return; + } + else + { + m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError, "Cannot remove hardlink"); + } + } + RenameOutputFile(filename, destDir); } } diff --git a/daemon/postprocess/PrePostProcessor.cpp b/daemon/postprocess/PrePostProcessor.cpp index 56aaeb1d..07f0a9da 100644 --- a/daemon/postprocess/PrePostProcessor.cpp +++ b/daemon/postprocess/PrePostProcessor.cpp @@ -506,6 +506,13 @@ void PrePostProcessor::DeleteCleanup(NzbInfo* nzbInfo) // delete old directory (if empty) FileSystem::DeleteDirectory(nzbInfo->GetDestDir()); + + // delete final directory (if empty) + const auto finalDir = nzbInfo->BuildFinalDirName(); + if (nzbInfo->GetDestDir() != finalDir && FileSystem::DirectoryExists(finalDir)) + { + FileSystem::DeleteDirectory(finalDir); + } } } diff --git a/daemon/queue/DirectRenamer.cpp b/daemon/queue/DirectRenamer.cpp index c6218364..6e5de99f 100644 --- a/daemon/queue/DirectRenamer.cpp +++ b/daemon/queue/DirectRenamer.cpp @@ -415,7 +415,12 @@ int DirectRenamer::RenameFilesInProgress(NzbInfo* nzbInfo, FileHashList* parHash newName = BuildNewRegularName(fileInfo->GetFilename(), parHashes, fileInfo->GetHash16k()); } - if (newName.empty()) + if (newName.empty() && !fileInfo->GetParFile() && g_Options->GetHardLinking()) + { + // Force rename if file is not par file and hardlinking is on + newName = fileInfo->GetFilename(); + } + else if (newName.empty()) { continue; } @@ -426,19 +431,44 @@ int DirectRenamer::RenameFilesInProgress(NzbInfo* nzbInfo, FileHashList* parHash const std::string oldOutputFilename = currOutputFilename.empty() ? nzbDestDir + PATH_SEPARATOR + fileInfo->GetFilename() : currOutputFilename; - const auto [_, newFilename] = FileSystem::SplitPathAndFilename(newOutputFilename); nzbInfo->PrintMessage(Message::mkInfo, "Renaming in-progress file %s to %s", oldOutputFilename.c_str(), newOutputFilename.c_str() ); + // Create hardlink and save the path in fileInfo + if (g_Options->GetHardLinking() && !Util::MatchFileExt(newName.c_str(), g_Options->GetHardLinkingIgnoreExt(), ",")) + { + const std::string nzbFinalDir = fileInfo->GetNzbInfo()->BuildFinalDirName().Str(); + const std::string finalOutputFilename = nzbFinalDir + PATH_SEPARATOR + newName; + + nzbInfo->PrintMessage(Message::mkInfo, + "HardLinking in-progress file %s to %s", + oldOutputFilename.c_str(), finalOutputFilename.c_str() + ); + + CString errmsg; + if (FileSystem::CreateHardLink(oldOutputFilename.c_str(), finalOutputFilename.c_str(), errmsg)) + { + fileInfo->SetHardLinkPath(finalOutputFilename); + } + else + { + nzbInfo->PrintMessage(Message::mkError, + "Could not create hardlink %s to %s: %s", + oldOutputFilename.c_str(), finalOutputFilename.c_str(), *errmsg + ); + } + } + if (Util::EmptyStr(fileInfo->GetOrigname())) { fileInfo->SetOrigname(fileInfo->GetFilename()); } - fileInfo->SetFilename(std::move(newFilename)); + // Set the filename to the full relative path + fileInfo->SetFilename(std::move(newName)); fileInfo->SetFilenameConfirmed(true); ++renamedFiles; } @@ -462,7 +492,12 @@ int DirectRenamer::RenameCompletedFiles(NzbInfo* nzbInfo, FileHashList* parHashe newName = BuildNewRegularName(completedFile.GetFilename(), parHashes, completedFile.GetHash16k()); } - if (newName.empty()) + if (newName.empty() && !completedFile.GetParFile() && g_Options->GetHardLinking()) + { + // Force rename if file is not par file and hardlinking is on + newName = completedFile.GetFilename(); + } + else if (newName.empty()) { continue; } @@ -470,20 +505,40 @@ int DirectRenamer::RenameCompletedFiles(NzbInfo* nzbInfo, FileHashList* parHashe const std::string destDir = nzbInfo->GetDestDir(); const std::string oldOutputFilename = destDir + PATH_SEPARATOR + completedFile.GetFilename(); const std::string outputFilename = destDir + PATH_SEPARATOR + newName; - const auto [_, newBaseName] = FileSystem::SplitPathAndFilename(outputFilename); nzbInfo->PrintMessage(Message::mkInfo, "Renaming completed file %s to %s", oldOutputFilename.c_str(), outputFilename.c_str() ); + if (!Util::EmptyStr(g_Options->GetInterDir()) && g_Options->GetHardLinking() && !Util::MatchFileExt(newName.c_str(), g_Options->GetHardLinkingIgnoreExt(), ",")) + { + const std::string finalDir = nzbInfo->BuildFinalDirName().Str(); + const std::string finalOutputFilename = finalDir + PATH_SEPARATOR + newName; + + nzbInfo->PrintMessage(Message::mkInfo, + "HardLinking completed file %s to %s", + oldOutputFilename.c_str(), finalOutputFilename.c_str() + ); + + CString errmsg; + if (!FileSystem::CreateHardLink(oldOutputFilename.c_str(), finalOutputFilename.c_str(), errmsg)) + { + nzbInfo->PrintMessage(Message::mkError, + "Could not create hardlink %s to %s: %s", + oldOutputFilename.c_str(), finalOutputFilename.c_str(), *errmsg + ); + } + } + if (RenameFile(nzbInfo, oldOutputFilename, outputFilename)) { if (Util::EmptyStr(completedFile.GetOrigname())) { completedFile.SetOrigname(completedFile.GetFilename()); } - completedFile.SetFilename(std::move(newBaseName)); + // Set the filename to the full relative path + completedFile.SetFilename(std::move(newName)); ++renamedFiles; } } diff --git a/daemon/queue/DownloadInfo.cpp b/daemon/queue/DownloadInfo.cpp index 5e899630..054e145f 100644 --- a/daemon/queue/DownloadInfo.cpp +++ b/daemon/queue/DownloadInfo.cpp @@ -838,6 +838,11 @@ void FileInfo::SetActiveDownloads(int activeDownloads) } } +bool FileInfo::IsHardLinked() +{ + return !m_hardLinkPath.empty(); +} + CompletedFile::CompletedFile(int id, std::string filename, std::string origname, EStatus status, uint32 crc, bool parFile, std::string hash16k, std::string parSetId) diff --git a/daemon/queue/DownloadInfo.h b/daemon/queue/DownloadInfo.h index a1aab8f5..746f8dde 100644 --- a/daemon/queue/DownloadInfo.h +++ b/daemon/queue/DownloadInfo.h @@ -205,6 +205,9 @@ class FileInfo void SetParSetId(const char* parSetId) { m_parSetId = parSetId; } bool GetFlushLocked() { return m_flushLocked; } void SetFlushLocked(bool flushLocked) { m_flushLocked = flushLocked; } + const std::string& GetHardLinkPath() const { return m_hardLinkPath; } + void SetHardLinkPath(std::string hardLinkPath) { m_hardLinkPath = std::move(hardLinkPath); } + bool IsHardLinked(); ServerStatList* GetServerStats() { return &m_serverStats; } @@ -246,6 +249,7 @@ class FileInfo CString m_hash16k; CString m_parSetId; bool m_flushLocked = false; + std::string m_hardLinkPath; static int m_idGen; static int m_idMax; diff --git a/daemon/queue/QueueCoordinator.cpp b/daemon/queue/QueueCoordinator.cpp index 5b05183a..9157125f 100644 --- a/daemon/queue/QueueCoordinator.cpp +++ b/daemon/queue/QueueCoordinator.cpp @@ -920,6 +920,10 @@ void QueueCoordinator::DiscardTempFiles(FileInfo* fileInfo) if (g_Options->GetDirectWrite() && !outputFilename.empty() && !fileInfo->GetForceDirectWrite()) { FileSystem::DeleteFile(outputFilename.c_str()); + if (fileInfo->IsHardLinked()) + { + FileSystem::DeleteFile(fileInfo->GetHardLinkPath().c_str()); + } } } diff --git a/daemon/util/FileSystem.cpp b/daemon/util/FileSystem.cpp index e5bccfa8..708cf179 100644 --- a/daemon/util/FileSystem.cpp +++ b/daemon/util/FileSystem.cpp @@ -230,6 +230,31 @@ bool FileSystem::ForceDirectories(const char* path, CString& errmsg) } #endif +bool FileSystem::CreateHardLink(const char *from, const char *to, CString& errmsg) +{ + const auto [newPath, _] = SplitPathAndFilename(to); + if (!ForceDirectories(newPath.c_str(), errmsg)) + { + return false; + } + +#ifdef WIN32 + if (!CreateHardLinkW(UtfPathToWidePath(to), UtfPathToWidePath(from), nullptr)) + { + errmsg = GetLastErrorMessage(); + return false; + } +#else + if (link(from, to) != 0) + { + errmsg = GetLastErrorMessage(); + return false; + } +#endif + + return true; +} + CString FileSystem::GetCurrentDirectory() { #ifdef WIN32 @@ -710,6 +735,14 @@ bool FileSystem::DeleteDirectory(const char* dirFilename) DirBrowser dir(dirFilename); while (const char* filename = dir.Next()) { + BString<1024> fullFilename("%s%c%s", dirFilename, PATH_SEPARATOR, filename); + + // Recursivly remove empty folder, useful in case of abort with direct rename and hardlinking + if (DirectoryExists(fullFilename) && DeleteDirectory(fullFilename)) + { + continue; + } + if (*filename != '.') { // calling RemoveDirectory to set correct errno diff --git a/daemon/util/FileSystem.h b/daemon/util/FileSystem.h index 76aa0695..898b9ff1 100644 --- a/daemon/util/FileSystem.h +++ b/daemon/util/FileSystem.h @@ -67,6 +67,7 @@ class FileSystem static bool DeleteDirectoryWithContent(const char* dirFilename, CString& errmsg); static bool ForceDirectories(const char* path, CString& errmsg); + static bool CreateHardLink(const char *from, const char *to, CString& errmsg); static CString GetCurrentDirectory(); static bool SetCurrentDirectory(const char* dirFilename); static int64 FileSize(const char* filename); diff --git a/nzbget.conf b/nzbget.conf index 3cf2e5b4..c8a99390 100644 --- a/nzbget.conf +++ b/nzbget.conf @@ -1444,6 +1444,24 @@ RarRename=yes # downloads. DirectRename=no +# Hardlink files during downloading (yes, no). +# +# Hardlink the final files while dowloading in . +# and must be located on the same file system. +# Useful for streaming media from usenet. +# +# NOTE: Only needed if is enabled. +# +# NOTE: On Windows only NTFS file system is supported. +HardLinking=no + +# File extensions to ignore during hardlinking. +# +# Example: .zip, .7z, .rar, *.7z.###, *.r## +# +# NOTE: Only needed if is enabled. +HardLinkingIgnoreExt=.zip, .7z, .rar, *.7z.###, *.r## + # What to do if download health drops below critical health (delete, park, # pause, none). #