serve/http: add --disable-dir-list flag

Previously, GET requests for a directory URL always returned an HTML
directory listing. There was no way to suppress this, unlike
`serve webdav` which has supported --disable-dir-list since #4191.

This adds the same flag to `serve http`. When set, GET requests for
directory URLs return 404 instead of a listing, while file downloads
continue to work normally.

Based on the approach suggested in #6306.

Fixes #4000

cmd/serve/http/http.go

@@ -38,6 +38,10 @@ var OptionsInfo = fs.Options{{
 	Name:    "disable_zip",
 	Default: false,
 	Help:    "Disable zip download of directories",
+}, {
+	Name:    "disable_dir_list",
+	Default: false,
+	Help:    "Disable HTML directory list on GET request for a directory",
 }}.
 	Add(libhttp.ConfigInfo).
 	Add(libhttp.AuthConfigInfo).
@@ -49,6 +53,7 @@ type Options struct {
 	HTTP           libhttp.Config
 	Template       libhttp.TemplateConfig
 	DisableZip     bool `config:"disable_zip"`
+	DisableDirList bool `config:"disable_dir_list"`
 }
 
 // DefaultOpt is the default values used for Options
@@ -259,6 +264,10 @@ func (s *HTTP) handler(w http.ResponseWriter, r *http.Request) {
 	isDir := strings.HasSuffix(r.URL.Path, "/")
 	remote := strings.Trim(r.URL.Path, "/")
 	if isDir {
+		if s.opt.DisableDirList {
+			http.NotFound(w, r)
+			return
+		}
 		s.serveDir(w, r, remote)
 	} else {
 		s.serveFile(w, r, remote)
@@ -347,6 +356,12 @@ func (s *HTTP) serveFile(w http.ResponseWriter, r *http.Request, remote string)
 	node, err := VFS.Stat(remote)
 	if err == vfs.ENOENT {
 		fs.Infof(remote, "%s: File not found", r.RemoteAddr)
+		if s.opt.DisableDirList {
+			// Return the same response as for a directory URL so
+			// that missing and existing paths are indistinguishable
+			http.NotFound(w, r)
+			return
+		}
 		http.Error(w, "File not found", http.StatusNotFound)
 		return
 	} else if err != nil {
@@ -354,6 +369,13 @@ func (s *HTTP) serveFile(w http.ResponseWriter, r *http.Request, remote string)
 		return
 	}
 	if !node.IsFile() {
+		if s.opt.DisableDirList {
+			// Return the same response as for a directory URL so
+			// that a directory's existence can't be probed via a
+			// URL without a trailing slash
+			http.NotFound(w, r)
+			return
+		}
 		http.Error(w, "Not a file", http.StatusNotFound)
 		return
 	}

cmd/serve/http/http_test.go

@@ -442,6 +442,100 @@ func TestCompressedTextFile(t *testing.T) {
 	assert.Equal(t, "0123456789\n", string(body))
 }
 
+func TestDisableDirList(t *testing.T) {
+	ctx := context.Background()
+	require.NoError(t, setAllModTimes("testdata/files", expectedTime))
+	f, err := fs.NewFs(ctx, "testdata/files")
+	require.NoError(t, err)
+
+	do := func(t *testing.T, disableDirList bool, url string) (int, []byte) {
+		t.Helper()
+		opts := Options{
+			HTTP: libhttp.DefaultCfg(),
+			Template: libhttp.TemplateConfig{
+				Path: testTemplate,
+			},
+			DisableDirList: disableDirList,
+		}
+		opts.HTTP.ListenAddr = []string{testBindAddress}
+		opts.Auth.BasicUser = testUser
+		opts.Auth.BasicPass = testPass
+
+		s, err := newServer(ctx, f, &opts, &vfscommon.Opt, &proxy.Opt)
+		require.NoError(t, err)
+		go func() { require.NoError(t, s.Serve()) }()
+		defer func() { assert.NoError(t, s.server.Shutdown()) }()
+
+		urls := s.server.URLs()
+		require.Len(t, urls, 1)
+		testURL := urls[0]
+
+		pause := time.Millisecond
+		for range 10 {
+			resp, err := http.Head(testURL)
+			if err == nil {
+				_ = resp.Body.Close()
+				break
+			}
+			time.Sleep(pause)
+			pause *= 2
+		}
+
+		req, err := http.NewRequest("GET", testURL+url, nil)
+		require.NoError(t, err)
+		req.SetBasicAuth(testUser, testPass)
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer func() { _ = resp.Body.Close() }()
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		return resp.StatusCode, body
+	}
+
+	t.Run("enabled by default - root dir lists", func(t *testing.T) {
+		status, body := do(t, false, "")
+		assert.Equal(t, http.StatusOK, status)
+		assert.Contains(t, string(body), "Directory listing of /")
+	})
+
+	t.Run("enabled by default - subdir lists", func(t *testing.T) {
+		status, body := do(t, false, "three/")
+		assert.Equal(t, http.StatusOK, status)
+		assert.Contains(t, string(body), "Directory listing of /three")
+	})
+
+	t.Run("disabled - root dir returns not found", func(t *testing.T) {
+		status, _ := do(t, true, "")
+		assert.Equal(t, http.StatusNotFound, status)
+	})
+
+	t.Run("disabled - subdir returns not found", func(t *testing.T) {
+		status, _ := do(t, true, "three/")
+		assert.Equal(t, http.StatusNotFound, status)
+	})
+
+	t.Run("disabled - existing and non-existent dirs return identical response", func(t *testing.T) {
+		// All must return the same status and body to prevent
+		// directory enumeration, with or without a trailing slash
+		for _, dir := range []string{"three/", "doesnotexist/", "three", "doesnotexist"} {
+			status, body := do(t, true, dir)
+			assert.Equal(t, http.StatusNotFound, status, dir)
+			assert.Equal(t, "404 page not found\n", string(body), dir)
+		}
+	})
+
+	t.Run("disabled - files still served", func(t *testing.T) {
+		status, body := do(t, true, "two.txt")
+		assert.Equal(t, http.StatusOK, status)
+		assert.Equal(t, "0123456789\n", string(body))
+	})
+
+	t.Run("disabled - subdir file still served", func(t *testing.T) {
+		status, _ := do(t, true, "three/a.txt")
+		assert.Equal(t, http.StatusOK, status)
+	})
+}
+
 func TestRc(t *testing.T) {
 	servetest.TestRc(t, rc.Params{
 		"type":           "http",