diff --git a/config/tomlloader.go b/config/tomlloader.go
index 95273ece..c38a983e 100644
--- a/config/tomlloader.go
+++ b/config/tomlloader.go
@@ -153,9 +153,8 @@ func setDefaultIfEmpty(server *ServerInfo, d ServerInfo) error {
 		}
 
 		if server.User == "" {
-			if Conf.Default.User != "" {
-				server.User = Conf.Default.User
-			} else {
+			server.User = Conf.Default.User
+			if server.User == "" && server.Port != "local" {
 				return xerrors.Errorf("server.user is empty")
 			}
 		}