Hardlinking while direct rename

daemon/main/Options.cpp

@@ -345,6 +345,8 @@ void Options::InitDefaults()
 	SetOption(RARRENAME.data(), "yes");
 	SetOption(HEALTHCHECK.data(), "none");
 	SetOption(DIRECTRENAME.data(), "no");
+	SetOption(HARDLINKING.data(), "no");
+	SetOption(HARDLINKINGIGNOREEXT.data(), ".zip, .7z, .rar, *.7z.###, *.r##");
 	SetOption(SCRIPTORDER.data(), "");
 	SetOption(EXTENSIONS.data(), "");
 	SetOption(DAEMONUSERNAME.data(), "root");
@@ -637,6 +639,8 @@ void Options::InitOptions()
 	m_parRename				= (bool)ParseEnumValue(PARRENAME.data(), BoolCount, BoolNames, BoolValues);
 	m_rarRename				= (bool)ParseEnumValue(RARRENAME.data(), BoolCount, BoolNames, BoolValues);
 	m_directRename			= (bool)ParseEnumValue(DIRECTRENAME.data(), BoolCount, BoolNames, BoolValues);
+	m_hardLinking			= (bool)ParseEnumValue(HARDLINKING.data(), BoolCount, BoolNames, BoolValues);
+	m_hardLinkingIgnoreExt = GetOption(HARDLINKINGIGNOREEXT.data());
 	m_cursesNzbName			= (bool)ParseEnumValue(CURSESNZBNAME.data(), BoolCount, BoolNames, BoolValues);
 	m_cursesTime			= (bool)ParseEnumValue(CURSESTIME.data(), BoolCount, BoolNames, BoolValues);
 	m_cursesGroup			= (bool)ParseEnumValue(CURSESGROUP.data(), BoolCount, BoolNames, BoolValues);

daemon/main/Options.h

@@ -106,6 +106,8 @@ class Options
 	static constexpr std::string_view RARRENAME = "RarRename";
 	static constexpr std::string_view HEALTHCHECK = "HealthCheck";
 	static constexpr std::string_view DIRECTRENAME = "DirectRename";
+	static constexpr std::string_view HARDLINKING = "HardLinking";
+	static constexpr std::string_view HARDLINKINGIGNOREEXT = "HardLinkingIgnoreExt";
 	static constexpr std::string_view UMASK = "UMask";
 	static constexpr std::string_view UPDATEINTERVAL = "UpdateInterval";
 	static constexpr std::string_view CURSESNZBNAME = "CursesNzbName";
@@ -484,6 +486,8 @@ class Options
 	int GetQuotaStartDay() const { return m_quotaStartDay; }
 	int GetDailyQuota() const { return m_dailyQuota; }
 	bool GetDirectRename() const { return m_directRename; }
+	bool GetHardLinking() const { return m_hardLinking; }
+	const char* GetHardLinkingIgnoreExt() const { return m_hardLinkingIgnoreExt; }
 	bool GetReorderFiles() const { return m_reorderFiles; }
 	EFileNaming GetFileNaming() const { return m_fileNaming; }
 	int GetDownloadRate() const { return m_downloadRate; }
@@ -619,6 +623,8 @@ class Options
 	int m_parThreads = 0;
 	bool m_rarRename = false;
 	bool m_directRename = false;
+	bool m_hardLinking = false;
+	CString m_hardLinkingIgnoreExt;
 	EHealthCheck m_healthCheck = hcNone;
 	CString m_extensions;
 	CString m_scriptOrder;

daemon/nntp/ArticleWriter.cpp

@@ -583,6 +583,20 @@ void ArticleWriter::CompleteFileParts()
 		GuardedDownloadQueue guard = DownloadQueue::Guard();
 		m_fileInfo->SetCrc(crc);
 
+		if (m_fileInfo->IsHardLinked() && Util::EmptyStr(g_Options->GetInterDir()))
+		{
+			// No need to rename, remove hardlink of .out.tmp-file
+			if (FileSystem::DeleteFile(m_outputFilename.c_str()))
+			{
+				m_fileInfo->SetOutputFilename(nullptr);
+				return;
+			}
+			else
+			{
+				m_fileInfo->GetNzbInfo()->PrintMessage(Message::mkError, "Cannot remove hardlink");
+			}
+		}
+
 		RenameOutputFile(filename, destDir);
 	}
 }

daemon/postprocess/PrePostProcessor.cpp

@@ -506,6 +506,13 @@ void PrePostProcessor::DeleteCleanup(NzbInfo* nzbInfo)
 
 		// delete old directory (if empty)
 		FileSystem::DeleteDirectory(nzbInfo->GetDestDir());
+
+		// delete final directory (if empty)
+		const auto finalDir = nzbInfo->BuildFinalDirName();
+		if (nzbInfo->GetDestDir() != finalDir && FileSystem::DirectoryExists(finalDir))
+		{
+			FileSystem::DeleteDirectory(finalDir);
+		}
 	}
 }
 

daemon/queue/DirectRenamer.cpp

@@ -415,7 +415,12 @@ int DirectRenamer::RenameFilesInProgress(NzbInfo* nzbInfo, FileHashList* parHash
 			newName = BuildNewRegularName(fileInfo->GetFilename(), parHashes, fileInfo->GetHash16k());
 		}
 
-		if (newName.empty())
+		if (newName.empty() && !fileInfo->GetParFile() && g_Options->GetHardLinking())
+		{
+			// Force rename if file is not par file and hardlinking is on
+			newName = fileInfo->GetFilename();
+		}
+		else if (newName.empty())
 		{
 			continue;
 		}
@@ -426,19 +431,44 @@ int DirectRenamer::RenameFilesInProgress(NzbInfo* nzbInfo, FileHashList* parHash
 		const std::string oldOutputFilename = currOutputFilename.empty()
 			? nzbDestDir + PATH_SEPARATOR + fileInfo->GetFilename()
 			: currOutputFilename;
-		const auto [_, newFilename] = FileSystem::SplitPathAndFilename(newOutputFilename);
 
 		nzbInfo->PrintMessage(Message::mkInfo,
 			"Renaming in-progress file %s to %s",
 			oldOutputFilename.c_str(), newOutputFilename.c_str()
 		);
 
+		// Create hardlink and save the path in fileInfo
+		if (g_Options->GetHardLinking() && !Util::MatchFileExt(newName.c_str(), g_Options->GetHardLinkingIgnoreExt(), ","))
+		{
+			const std::string nzbFinalDir = fileInfo->GetNzbInfo()->BuildFinalDirName().Str();
+			const std::string finalOutputFilename = nzbFinalDir + PATH_SEPARATOR + newName;
+
+			nzbInfo->PrintMessage(Message::mkInfo,
+				"HardLinking in-progress file %s to %s",
+				oldOutputFilename.c_str(), finalOutputFilename.c_str()
+			);
+
+			CString errmsg;
+			if (FileSystem::CreateHardLink(oldOutputFilename.c_str(), finalOutputFilename.c_str(), errmsg))
+			{
+				fileInfo->SetHardLinkPath(finalOutputFilename);
+			}
+			else
+			{
+				nzbInfo->PrintMessage(Message::mkError,
+					"Could not create hardlink %s to %s: %s",
+					oldOutputFilename.c_str(), finalOutputFilename.c_str(), *errmsg
+				);
+			}
+		}
+
 		if (Util::EmptyStr(fileInfo->GetOrigname()))
 		{
 			fileInfo->SetOrigname(fileInfo->GetFilename());
 		}
 
-		fileInfo->SetFilename(std::move(newFilename));
+		// Set the filename to the full relative path
+		fileInfo->SetFilename(std::move(newName));
 		fileInfo->SetFilenameConfirmed(true);
 		++renamedFiles;
 	}
@@ -462,28 +492,53 @@ int DirectRenamer::RenameCompletedFiles(NzbInfo* nzbInfo, FileHashList* parHashe
 			newName = BuildNewRegularName(completedFile.GetFilename(), parHashes, completedFile.GetHash16k());
 		}
 
-		if (newName.empty())
+		if (newName.empty() && !completedFile.GetParFile() && g_Options->GetHardLinking())
+		{
+			// Force rename if file is not par file and hardlinking is on
+			newName = completedFile.GetFilename();
+		}
+		else if (newName.empty())
 		{
 			continue;
 		}
 
 		const std::string destDir = nzbInfo->GetDestDir();
 		const std::string oldOutputFilename = destDir + PATH_SEPARATOR + completedFile.GetFilename();
 		const std::string outputFilename = destDir + PATH_SEPARATOR + newName;
-		const auto [_, newBaseName] = FileSystem::SplitPathAndFilename(outputFilename);
 
 		nzbInfo->PrintMessage(Message::mkInfo,
 			"Renaming completed file %s to %s",
 			oldOutputFilename.c_str(), outputFilename.c_str()
 		);
 
+		if (!Util::EmptyStr(g_Options->GetInterDir()) && g_Options->GetHardLinking() && !Util::MatchFileExt(newName.c_str(), g_Options->GetHardLinkingIgnoreExt(), ","))
+		{
+			const std::string finalDir = nzbInfo->BuildFinalDirName().Str();
+			const std::string finalOutputFilename = finalDir + PATH_SEPARATOR + newName;
+
+			nzbInfo->PrintMessage(Message::mkInfo,
+				"HardLinking completed file %s to %s",
+				oldOutputFilename.c_str(), finalOutputFilename.c_str()
+			);
+
+			CString errmsg;
+			if (!FileSystem::CreateHardLink(oldOutputFilename.c_str(), finalOutputFilename.c_str(), errmsg))
+			{
+				nzbInfo->PrintMessage(Message::mkError,
+					"Could not create hardlink %s to %s: %s",
+					oldOutputFilename.c_str(), finalOutputFilename.c_str(), *errmsg
+				);
+			}
+		}
+
 		if (RenameFile(nzbInfo, oldOutputFilename, outputFilename))
 		{
 			if (Util::EmptyStr(completedFile.GetOrigname()))
 			{
 				completedFile.SetOrigname(completedFile.GetFilename());
 			}
-			completedFile.SetFilename(std::move(newBaseName));
+			// Set the filename to the full relative path
+			completedFile.SetFilename(std::move(newName));
 			++renamedFiles;
 		}
 	}

daemon/queue/DownloadInfo.cpp

@@ -838,6 +838,11 @@ void FileInfo::SetActiveDownloads(int activeDownloads)
 	}
 }
 
+bool FileInfo::IsHardLinked()
+{
+	return !m_hardLinkPath.empty();
+}
+
 
 CompletedFile::CompletedFile(int id, std::string filename, std::string origname, EStatus status,
 	uint32 crc, bool parFile, std::string hash16k, std::string parSetId) 

daemon/queue/DownloadInfo.h

@@ -205,6 +205,9 @@ class FileInfo
 	void SetParSetId(const char* parSetId) { m_parSetId = parSetId; }
 	bool GetFlushLocked() { return m_flushLocked; }
 	void SetFlushLocked(bool flushLocked) { m_flushLocked = flushLocked; }
+	const std::string& GetHardLinkPath() const { return m_hardLinkPath; }
+	void SetHardLinkPath(std::string hardLinkPath) { m_hardLinkPath = std::move(hardLinkPath); }
+	bool IsHardLinked();
 
 	ServerStatList* GetServerStats() { return &m_serverStats; }
 
@@ -246,6 +249,7 @@ class FileInfo
 	CString m_hash16k;
 	CString m_parSetId;
 	bool m_flushLocked = false;
+	std::string m_hardLinkPath;
 
 	static int m_idGen;
 	static int m_idMax;

daemon/queue/QueueCoordinator.cpp

@@ -920,6 +920,10 @@ void QueueCoordinator::DiscardTempFiles(FileInfo* fileInfo)
 	if (g_Options->GetDirectWrite() && !outputFilename.empty() && !fileInfo->GetForceDirectWrite())
 	{
 		FileSystem::DeleteFile(outputFilename.c_str());
+		if (fileInfo->IsHardLinked())
+		{
+			FileSystem::DeleteFile(fileInfo->GetHardLinkPath().c_str());
+		}
 	}
 }
 

daemon/util/FileSystem.cpp

@@ -230,6 +230,31 @@ bool FileSystem::ForceDirectories(const char* path, CString& errmsg)
 }
 #endif
 
+bool FileSystem::CreateHardLink(const char *from, const char *to, CString& errmsg)
+{
+	const auto [newPath, _] = SplitPathAndFilename(to);
+	if (!ForceDirectories(newPath.c_str(), errmsg))
+	{
+		return false;
+	}
+
+#ifdef WIN32
+	if (!CreateHardLinkW(UtfPathToWidePath(to), UtfPathToWidePath(from), nullptr))
+	{
+		errmsg = GetLastErrorMessage();
+		return false;
+	}
+#else
+	if (link(from, to) != 0)
+	{
+		errmsg = GetLastErrorMessage();
+		return false;
+	}
+#endif
+
+	return true;
+}
+
 CString FileSystem::GetCurrentDirectory()
 {
 #ifdef WIN32
@@ -710,6 +735,14 @@ bool FileSystem::DeleteDirectory(const char* dirFilename)
 		DirBrowser dir(dirFilename);
 		while (const char* filename = dir.Next())
 		{
+			BString<1024> fullFilename("%s%c%s", dirFilename, PATH_SEPARATOR, filename);
+
+			// Recursivly remove empty folder, useful in case of abort with direct rename and hardlinking
+			if (DirectoryExists(fullFilename) && DeleteDirectory(fullFilename))
+			{
+				continue;
+			}
+
 			if (*filename != '.')
 			{
 				// calling RemoveDirectory to set correct errno

daemon/util/FileSystem.h

@@ -67,6 +67,7 @@ class FileSystem
 
 	static bool DeleteDirectoryWithContent(const char* dirFilename, CString& errmsg);
 	static bool ForceDirectories(const char* path, CString& errmsg);
+	static bool CreateHardLink(const char *from, const char *to, CString& errmsg);
 	static CString GetCurrentDirectory();
 	static bool SetCurrentDirectory(const char* dirFilename);
 	static int64 FileSize(const char* filename);

nzbget.conf

@@ -1444,6 +1444,24 @@ RarRename=yes
 # downloads.
 DirectRename=no
 
+# Hardlink files during downloading (yes, no).
+#
+#  Hardlink the final files while dowloading in <DestDir>.
+#  <InterDir> and <DestDir> must be located on the same file system.
+#  Useful for streaming media from usenet.
+#
+# NOTE: Only needed if <DirectRename> is enabled.
+#
+# NOTE: On Windows only NTFS file system is supported.
+HardLinking=no
+
+# File extensions to ignore during hardlinking.
+#
+# Example: .zip, .7z, .rar, *.7z.###, *.r##
+#
+# NOTE: Only needed if <HardLinking> is enabled.
+HardLinkingIgnoreExt=.zip, .7z, .rar, *.7z.###, *.r##
+
 # What to do if download health drops below critical health (delete, park,
 # pause, none).
 #