IO-less dirty throttling v11

+ * (o) bdi control lines
+ *
+ * The control lines for the global/bdi setpoints both stretch up to @limit.
+ * The below figure illustrates the main bdi control line with an auxiliary
+ * line extending it to @limit.
+ *
+ * o
+ * o
+ * o [o] main control line
+ * o [*] auxiliary control line
+ * o
+ * o
+ * o
+ * o
+ * o
+ * o

+ *                       o--------------------- balance point, rate scale = 1
+ *                       | o
+ *                       |   o
+ *                       |     o
+ *                       |       o
+ *                       |         o
+ *                       |           o
+ *                       |             o------- connect point, rate scale = 1/2
+ *                       |               .*
+ *                       |                 .   *
+ *                       |                   .      *
+ *                       |                     .         *
+ *                       |                       .           *
+ *                       |                         .              *
+ *                       |                           .                 *

+ * [--------------------+-----------------------------.--------------------*]
+ * 0 bdi_setpoint x_intercept limit
+ *
+ * The auxiliary control line allows smoothly throttling bdi_dirty down to
+ * normal if it starts high in situations like
+ * - start writing to a slow SD card and a fast disk at the same time. The SD
+ * card's bdi_dirty may rush to many times higher than bdi_setpoint.
+ * - the bdi dirty thresh drops quickly due to change of JBOD workload

Quoted 31 lines · Wu Fengguang

+ * (o) bdi control lines
+ *
+ * The control lines for the global/bdi setpoints both stretch up to @limit.
+ * The below figure illustrates the main bdi control line with an auxiliary
+ * line extending it to @limit.
+ *
+ * o
+ * o
+ * o [o] main control line
+ * o [*] auxiliary control line
+ * o
+ * o
+ * o
+ * o
+ * o
+ * o

+ *                       o--------------------- balance point, rate scale = 1
+ *                       | o
+ *                       |   o
+ *                       |     o
+ *                       |       o
+ *                       |         o
+ *                       |           o
+ *                       |             o------- connect point, rate scale = 1/2
+ *                       |               .*
+ *                       |                 .   *
+ *                       |                   .      *
+ *                       |                     .         *
+ *                       |                       .           *
+ *                       |                         .              *
+ *                       |                           .                 *

Quoted 8 lines

+ * [--------------------+-----------------------------.--------------------*]
+ * 0 bdi_setpoint x_intercept limit
+ *
+ * The auxiliary control line allows smoothly throttling bdi_dirty down to
+ * normal if it starts high in situations like
+ * - start writing to a slow SD card and a fast disk at the same time. The SD
+ * card's bdi_dirty may rush to many times higher than bdi_setpoint.
+ * - the bdi dirty thresh drops quickly due to change of JBOD workload

In light of the global control thing already having a hard stop at limit, what's the point of the auxiliary line? Why not simply run the bdi control between [0.5, 1.5] and leave it at that?

mm/page-writeback.c +9 -5

@@ -591,6 +790,7 @@ static void global_update_bandwidth(unsi
 
 void __bdi_update_bandwidth(struct backing_dev_info *bdi,
                            unsigned long thresh,
+                           unsigned long bg_thresh,
                            unsigned long dirty,
                            unsigned long bdi_thresh,
                            unsigned long bdi_dirty,

@@ -627,6 +827,7 @@ snapshot:
 
 static void bdi_update_bandwidth(struct backing_dev_info *bdi,
                                 unsigned long thresh,
+                                unsigned long bg_thresh,
                                 unsigned long dirty,
                                 unsigned long bdi_thresh,
                                 unsigned long bdi_dirty,

@@ -635,8 +836,8 @@ static void bdi_update_bandwidth(struct 
        if (time_is_after_eq_jiffies(bdi->bw_time_stamp + BANDWIDTH_INTERVAL))
                return;
        spin_lock(&bdi->wb.list_lock);
-       __bdi_update_bandwidth(bdi, thresh, dirty, bdi_thresh, bdi_dirty,
-                              start_time);
+       __bdi_update_bandwidth(bdi, thresh, bg_thresh, dirty,
+                              bdi_thresh, bdi_dirty, start_time);
        spin_unlock(&bdi->wb.list_lock);
 }
 

@@ -677,7 +878,8 @@ static void balance_dirty_pages(struct a
                 * catch-up. This avoids (excessively) small writeouts
                 * when the bdi limits are ramping up.
                 */
-               if (nr_dirty <= (background_thresh + dirty_thresh) / 2)
+               if (nr_dirty <= dirty_freerun_ceiling(dirty_thresh,
+                                                     background_thresh))
                        break;
 
                bdi_thresh = bdi_dirty_limit(bdi, dirty_thresh);

@@ -721,8 +923,9 @@ static void balance_dirty_pages(struct a
                if (!bdi->dirty_exceeded)
                        bdi->dirty_exceeded = 1;
 
-               bdi_update_bandwidth(bdi, dirty_thresh, nr_dirty,
-                                    bdi_thresh, bdi_dirty, start_time);
+               bdi_update_bandwidth(bdi, dirty_thresh, background_thresh,
+                                    nr_dirty, bdi_thresh, bdi_dirty,
+                                    start_time);
 
                /* Note: nr_reclaimable denotes nr_dirty + nr_unstable.
                 * Unstable writes are a feature of certain networked

fs/fs-writeback.c 2011-08-26 15:57:20.000000000 +0800 renamed from fs/fs-writeback.c 2011-08-26 15:57:18.000000000 +0800 +1 -1

@@ -675,7 +675,7 @@ static inline bool over_bground_thresh(v
 static void wb_update_bandwidth(struct bdi_writeback *wb,
                                unsigned long start_time)
 {
-       __bdi_update_bandwidth(wb->bdi, 0, 0, 0, 0, start_time);
+       __bdi_update_bandwidth(wb->bdi, 0, 0, 0, 0, 0, start_time);
 }
 
 /*

Quoted 64 lines · Wu Fengguang

mm/page-writeback.c +9 -5

@@ -591,6 +790,7 @@ static void global_update_bandwidth(unsi
 
 void __bdi_update_bandwidth(struct backing_dev_info *bdi,
                            unsigned long thresh,
+                           unsigned long bg_thresh,
                            unsigned long dirty,
                            unsigned long bdi_thresh,
                            unsigned long bdi_dirty,

@@ -627,6 +827,7 @@ snapshot:
 
 static void bdi_update_bandwidth(struct backing_dev_info *bdi,
                                 unsigned long thresh,
+                                unsigned long bg_thresh,
                                 unsigned long dirty,
                                 unsigned long bdi_thresh,
                                 unsigned long bdi_dirty,

@@ -635,8 +836,8 @@ static void bdi_update_bandwidth(struct 
        if (time_is_after_eq_jiffies(bdi->bw_time_stamp + BANDWIDTH_INTERVAL))
                return;
        spin_lock(&bdi->wb.list_lock);
-       __bdi_update_bandwidth(bdi, thresh, dirty, bdi_thresh, bdi_dirty,
-                              start_time);
+       __bdi_update_bandwidth(bdi, thresh, bg_thresh, dirty,
+                              bdi_thresh, bdi_dirty, start_time);
        spin_unlock(&bdi->wb.list_lock);
 }
 

@@ -677,7 +878,8 @@ static void balance_dirty_pages(struct a
                 * catch-up. This avoids (excessively) small writeouts
                 * when the bdi limits are ramping up.
                 */
-               if (nr_dirty <= (background_thresh + dirty_thresh) / 2)
+               if (nr_dirty <= dirty_freerun_ceiling(dirty_thresh,
+                                                     background_thresh))
                        break;
 
                bdi_thresh = bdi_dirty_limit(bdi, dirty_thresh);

@@ -721,8 +923,9 @@ static void balance_dirty_pages(struct a
                if (!bdi->dirty_exceeded)
                        bdi->dirty_exceeded = 1;
 
-               bdi_update_bandwidth(bdi, dirty_thresh, nr_dirty,
-                                    bdi_thresh, bdi_dirty, start_time);
+               bdi_update_bandwidth(bdi, dirty_thresh, background_thresh,
+                                    nr_dirty, bdi_thresh, bdi_dirty,
+                                    start_time);
 
                /* Note: nr_reclaimable denotes nr_dirty + nr_unstable.
                 * Unstable writes are a feature of certain networked

fs/fs-writeback.c 2011-08-26 15:57:20.000000000 +0800 renamed from fs/fs-writeback.c 2011-08-26 15:57:18.000000000 +0800 +1 -1

@@ -675,7 +675,7 @@ static inline bool over_bground_thresh(v
 static void wb_update_bandwidth(struct bdi_writeback *wb,
                                unsigned long start_time)
 {
-       __bdi_update_bandwidth(wb->bdi, 0, 0, 0, 0, start_time);
+       __bdi_update_bandwidth(wb->bdi, 0, 0, 0, 0, 0, start_time);
 }
 
 /*

include/linux/writeback.h 2011-08-26 15:57:20.000000000 +0800 renamed from include/linux/writeback.h 2011-08-26 15:57:18.000000000 +0800 +0 -0

@@ -141,6 +141,7 @@ unsigned long bdi_dirty_limit(struct bac
 

include/linux/writeback.h 2011-08-26 15:57:20.000000000 +0800 unsigned long bdi_dirty_limit(struct bac · line 147

Quoted 6 lines

include/linux/writeback.h 2011-08-26 15:57:20.000000000 +0800 renamed from include/linux/writeback.h 2011-08-26 15:57:18.000000000 +0800 +1 -0

 void __bdi_update_bandwidth(struct backing_dev_info *bdi,
                            unsigned long thresh,
+                           unsigned long bg_thresh,
                            unsigned long dirty,
                            unsigned long bdi_thresh,
                            unsigned long bdi_dirty,

All this function signature muck doesn't seem immediately relevant to the introduction of bdi_position_ratio() since the new function isn't actually used.

- in memory tight systems, (1) becomes strong enough to squeeze dirty pages inside the control scope

- in large memory systems where the "gravity" of (1) for pulling the dirty pages to setpoint is too weak, (2) can back (1) up and drive dirty pages to bdi_setpoint ~= setpoint reasonably fast.

Unfortunately in JBOD setups, the fluctuation range of bdi threshold is related to memory size due to the interferences between disks. In this case, the bdi slope will be weighted sum of write_bw and bdi_thresh.

Given equations

span = x_intercept - bdi_setpoint
k = df/dx = - 1 / span

and the extremum values

span = bdi_thresh
dx = bdi_thresh

we get

        df = - dx / span = - 1.0

That means, when bdi_dirty deviates bdi_thresh up, pos_ratio and hence task ratelimit will fluctuate by -100%.

On Sun, Sep 04, 2011 at 09:53:07AM +0800, Wu Fengguang wrote:

[..]

Quoted 10 lines

- in memory tight systems, (1) becomes strong enough to squeeze dirty pages inside the control scope

- in large memory systems where the "gravity" of (1) for pulling the dirty pages to setpoint is too weak, (2) can back (1) up and drive dirty pages to bdi_setpoint ~= setpoint reasonably fast.

Unfortunately in JBOD setups, the fluctuation range of bdi threshold is related to memory size due to the interferences between disks. In this case, the bdi slope will be weighted sum of write_bw and bdi_thresh.

Can you please elaborate a little more that what changes in JBOD setup.

Quoted 11 lines

Given equations

span = x_intercept - bdi_setpoint
k = df/dx = - 1 / span

and the extremum values

span = bdi_thresh
dx = bdi_thresh

Quoted 6 lines

we get

        df = - dx / span = - 1.0

That means, when bdi_dirty deviates bdi_thresh up, pos_ratio and hence task ratelimit will fluctuate by -100%.

I am not sure I understand above calculation. I understood the part that for single bdi case, you want 12.5% varation of bdi_setpoint over a range of write_bw [SP-write_bw/2, SP+write_bw/2]. This requirement will lead to.

k = -1/8*write_bw

OR span = 8*write_bw, hence
k= -1/span

Now I missed the part that what is different in case of JBOD setup and how do you come up with values for that setup so that slope of bdi setpoint is sharper.

IIUC, in case of single bdi case you want to use k=-1/(8*write_bw) and in case of JBOD you want to use k=-1/(bdi_thresh)?

That means for single bdi case you want to trust bdi, write_bw but in case of JBOD you stop trusting that and just switch to bdi_thresh. Not sure what does it mean.

bdi_position_ratio() provides a scale factor to bdi->dirty_ratelimit, so that the resulted task rate limit can drive the dirty pages back to the global/bdi setpoints.

Old scheme is,
                                         |
                          free run area  |  throttle area ----------------------------------------+----------------------------> thresh^                  dirty pages

New scheme is,

 ^ task rate limit

 |
 |            *
 |             *
 |              *
 |[free run]      *      [smooth throttled]
 |                  *
 |                     *
 |                         *
 ..bdi->dirty_ratelimit..........*
 |                               .     *
 |                               .          *
 |                               .              *
 |                               .                 *
 |                               .                    *

 +-------------------------------.-----------------------*------------> setpoint^                  limit^  dirty pages

The slope of the bdi control line should be

1) large enough to pull the dirty pages to setpoint reasonably fast

2) small enough to avoid big fluctuations in the resulted pos_ratio and hence task ratelimit

Since the fluctuation range of the bdi dirty pages is typically observed to be within 1-second worth of data, the bdi control line's slope is selected to be a linear function of bdi write bandwidth, so that it can adapt to slow/fast storage devices well.

Assume the bdi control line

       pos_ratio = 1.0 + k * (dirty - bdi_setpoint)

where k is the negative slope.

If targeting for 12.5% fluctuation range in pos_ratio when dirty pages are fluctuating in range

       [bdi_setpoint - write_bw/2, bdi_setpoint + write_bw/2],

we get slope

       k = - 1 / (8 * write_bw)

Let pos_ratio(x_intercept) = 0, we get the parameter used in code:

       x_intercept = bdi_setpoint + 8 * write_bw

The global/bdi slopes are nicely complementing each other when the system has only one major bdi (indicated by bdi_thresh ~= thresh):

1) slope of global control line    => scaling to the control scope size
2) slope of main bdi control line  => scaling to the writeout bandwidth

so that

- in memory tight systems, (1) becomes strong enough to squeeze dirty pages inside the control scope

- in large memory systems where the "gravity" of (1) for pulling the dirty pages to setpoint is too weak, (2) can back (1) up and drive dirty pages to bdi_setpoint ~= setpoint reasonably fast.

Unfortunately in JBOD setups, the fluctuation range of bdi threshold is related to memory size due to the interferences between disks.  In this case, the bdi slope will be weighted sum of write_bw and bdi_thresh.

Given equations

       span = x_intercept - bdi_setpoint
       k = df/dx = - 1 / span

and the extremum values

       span = bdi_thresh
       dx = bdi_thresh

we get

       df = - dx / span = - 1.0

That means, when bdi_dirty deviates bdi_thresh up, pos_ratio and hence task ratelimit will fluctuate by -100%.

peter: use 3rd order polynomial for the global control line

CC Peter Zijlstra <[email protected]>

Acked-by Jan Kara <[email protected]>

Signed-off-by Wu Fengguang <[email protected]>

---
 fs/fs-writeback.c         |    2
 include/linux/writeback.h |    1
 mm/page-writeback.c       |  213 +++++++++++++++++++++++++++++++++++-

 3 files changed, 210 insertions(+), 6 deletions(-)

mm/page-writeback.c      2011-08-26 15:57:34.000000000 +0800 renamed from mm/page-writeback.c 2011-08-26 15:57:18.000000000 +0800 +208 -5

@@ -46,6 +46,8 @@
  */
  #define BANDWIDTH_INTERVAL     max(HZ/5, 1)
 
+#define RATELIMIT_CALC_SHIFT   10
+
  /*
  * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
  * will look to see if it needs to force writeback or throttling.

@@ -409,6 +411,12 @@ int bdi_set_max_ratio(struct backing_dev
  }
  EXPORT_SYMBOL(bdi_set_max_ratio);
 
+static unsigned long dirty_freerun_ceiling(unsigned long thresh,
+                                          unsigned long bg_thresh)
+{
+       return (thresh + bg_thresh) / 2;
+}
+
  static unsigned long hard_dirty_limit(unsigned long thresh)
  {
        return max(thresh, global_dirty_limit);

@@ -493,6 +501,197 @@ unsigned long bdi_dirty_limit(struct bac
        return bdi_dirty;
  }
 
+/*
+ * Dirty position control.
+ *
+ * (o) global/bdi setpoints
+ *
+ * We want the dirty pages be balanced around the global/bdi setpoints.
+ * When the number of dirty pages is higher/lower than the setpoint, the
+ * dirty position control ratio (and hence task dirty ratelimit) will be
+ * decreased/increased to bring the dirty pages back to the setpoint.
+ *
+ *     pos_ratio = 1 << RATELIMIT_CALC_SHIFT
+ *
+ *     if (dirty < setpoint) scale up   pos_ratio
+ *     if (dirty > setpoint) scale down pos_ratio
+ *
+ *     if (bdi_dirty < bdi_setpoint) scale up   pos_ratio
+ *     if (bdi_dirty > bdi_setpoint) scale down pos_ratio
+ *
+ *     task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT
+ *
+ * (o) global control line
+ *
+ *     ^ pos_ratio
+ *     |
+ *     |            |<===== global dirty control scope ======>|
+ * 2.0 .............*
+ *     |            .*
+ *     |            . *
+ *     |            .   *
+ *     |            .     *
+ *     |            .        *
+ *     |            .            *
+ * 1.0 ................................*
+ *     |            .                  .     *
+ *     |            .                  .          *
+ *     |            .                  .              *
+ *     |            .                  .                 *
+ *     |            .                  .                    *
+ *   0 +------------.------------------.----------------------*------------->
+ *           freerun^          setpoint^                 limit^   dirty pages
+ *
+ * (o) bdi control lines
+ *
+ * The control lines for the global/bdi setpoints both stretch up to @limit.
+ * The below figure illustrates the main bdi control line with an auxiliary
+ * line extending it to @limit.
+ *
+ *   o
+ *     o
+ *       o                                      [o] main control line
+ *         o                                    [*] auxiliary control line
+ *           o
+ *             o
+ *               o
+ *                 o
+ *                   o
+ *                     o
+ *                       o--------------------- balance point, rate scale = 1
+ *                       | o
+ *                       |   o
+ *                       |     o
+ *                       |       o
+ *                       |         o
+ *                       |           o
+ *                       |             o------- connect point, rate scale = 1/2
+ *                       |               .*
+ *                       |                 .   *
+ *                       |                   .      *
+ *                       |                     .         *
+ *                       |                       .           *
+ *                       |                         .              *
+ *                       |                           .                 *
+ *  [--------------------+-----------------------------.--------------------*]
+ *  0              bdi_setpoint                    x_intercept           limit
+ *
+ * The auxiliary control line allows smoothly throttling bdi_dirty down to
+ * normal if it starts high in situations like
+ * - start writing to a slow SD card and a fast disk at the same time. The SD
+ *   card's bdi_dirty may rush to many times higher than bdi_setpoint.
+ * - the bdi dirty thresh drops quickly due to change of JBOD workload
+ */
+static unsigned long bdi_position_ratio(struct backing_dev_info *bdi,
+                                       unsigned long thresh,
+                                       unsigned long bg_thresh,
+                                       unsigned long dirty,
+                                       unsigned long bdi_thresh,
+                                       unsigned long bdi_dirty)
+{
+       unsigned long write_bw = bdi->avg_write_bandwidth;
+       unsigned long freerun = dirty_freerun_ceiling(thresh, bg_thresh);
+       unsigned long limit = hard_dirty_limit(thresh);
+       unsigned long x_intercept;
+       unsigned long setpoint;         /* dirty pages' target balance point */
+       unsigned long bdi_setpoint;
+       unsigned long span;
+       long long pos_ratio;            /* for scaling up/down the rate limit */
+       long x;
+
+       if (unlikely(dirty >= limit))
+               return 0;
+
+       /*
+        * global setpoint
+        *
+        *                           setpoint - dirty 3
+        *        f(dirty) := 1.0 + (----------------)
+        *                           limit - setpoint
+        *
+        * it's a 3rd order polynomial that subjects to
+        *
+        * (1) f(freerun)  = 2.0 => rampup dirty_ratelimit reasonably fast
+        * (2) f(setpoint) = 1.0 => the balance point
+        * (3) f(limit)    = 0   => the hard limit
+        * (4) df/dx      <= 0   => negative feedback control
+        * (5) the closer to setpoint, the smaller |df/dx| (and the reverse)
+        *     => fast response on large errors; small oscillation near setpoint
+        */
+       setpoint = (freerun + limit) / 2;
+       x = div_s64((setpoint - dirty) << RATELIMIT_CALC_SHIFT,
+                   limit - setpoint + 1);
+       pos_ratio = x;
+       pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
+       pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
+       pos_ratio += 1 << RATELIMIT_CALC_SHIFT;
+
+       /*
+        * We have computed basic pos_ratio above based on global situation. If
+        * the bdi is over/under its share of dirty pages, we want to scale
+        * pos_ratio further down/up. That is done by the following mechanism.
+        */
+
+       /*
+        * bdi setpoint
+        *
+        *        f(bdi_dirty) := 1.0 + k * (bdi_dirty - bdi_setpoint)
+        *
+        *                        x_intercept - bdi_dirty
+        *                     := --------------------------
+        *                        x_intercept - bdi_setpoint
+        *
+        * The main bdi control line is a linear function that subjects to
+        *
+        * (1) f(bdi_setpoint) = 1.0
+        * (2) k = - 1 / (8 * write_bw)  (in single bdi case)
+        *     or equally: x_intercept = bdi_setpoint + 8 * write_bw
+        *
+        * For single bdi case, the dirty pages are observed to fluctuate
+        * regularly within range
+        *        [bdi_setpoint - write_bw/2, bdi_setpoint + write_bw/2]
+        * for various filesystems, where (2) can yield in a reasonable 12.5%
+        * fluctuation range for pos_ratio.
+        *
+        * For JBOD case, bdi_thresh (not bdi_dirty!) could fluctuate up to its
+        * own size, so move the slope over accordingly and choose a slope that
+        * yields 100% pos_ratio fluctuation on suddenly doubled bdi_thresh.
+        */
+       if (unlikely(bdi_thresh > thresh))
+               bdi_thresh = thresh;
+       /*
+        * scale global setpoint to bdi's:
+        *      bdi_setpoint = setpoint * bdi_thresh / thresh
+        */
+       x = div_u64((u64)bdi_thresh << 16, thresh + 1);
+       bdi_setpoint = setpoint * (u64)x >> 16;
+       /*
+        * Use span=(8*write_bw) in single bdi case as indicated by
+        * (thresh - bdi_thresh ~= 0) and transit to bdi_thresh in JBOD case.
+        *
+        *        bdi_thresh                    thresh - bdi_thresh
+        * span = ---------- * (8 * write_bw) + ------------------- * bdi_thresh
+        *          thresh                            thresh
+        */
+       span = (thresh - bdi_thresh + 8 * write_bw) * (u64)x >> 16;
+       x_intercept = bdi_setpoint + span;
+
+       span >>= 1;
+       if (unlikely(bdi_dirty > bdi_setpoint + span)) {
+               if (unlikely(bdi_dirty > limit))
+                       return 0;
+               if (x_intercept < limit) {
+                       x_intercept = limit;    /* auxiliary control line */
+                       bdi_setpoint += span;
+                       pos_ratio >>= 1;
+               }
+       }
+       pos_ratio *= x_intercept - bdi_dirty;
+       do_div(pos_ratio, x_intercept - bdi_setpoint + 1);
+
+       return pos_ratio;
+}
+
  static void bdi_update_write_bandwidth(struct backing_dev_info *bdi,
                                       unsigned long elapsed,
                                       unsigned long written)

@@ -591,6 +790,7 @@ static void global_update_bandwidth(unsi
 
  void __bdi_update_bandwidth(struct backing_dev_info *bdi,
                            unsigned long thresh,
+                           unsigned long bg_thresh,
                            unsigned long dirty,
                            unsigned long bdi_thresh,
                            unsigned long bdi_dirty,

@@ -627,6 +827,7 @@ snapshot:
 
  static void bdi_update_bandwidth(struct backing_dev_info *bdi,
                                 unsigned long thresh,
+                                unsigned long bg_thresh,
                                 unsigned long dirty,
                                 unsigned long bdi_thresh,
                                 unsigned long bdi_dirty,

@@ -635,8 +836,8 @@ static void bdi_update_bandwidth(struct
        if (time_is_after_eq_jiffies(bdi->bw_time_stamp + BANDWIDTH_INTERVAL))
                return;
        spin_lock(&bdi->wb.list_lock);
-       __bdi_update_bandwidth(bdi, thresh, dirty, bdi_thresh, bdi_dirty,
-                              start_time);
+       __bdi_update_bandwidth(bdi, thresh, bg_thresh, dirty,
+                              bdi_thresh, bdi_dirty, start_time);
        spin_unlock(&bdi->wb.list_lock);
  }
 

@@ -677,7 +878,8 @@ static void balance_dirty_pages(struct a
                 * catch-up. This avoids (excessively) small writeouts
                 * when the bdi limits are ramping up.
                 */
-               if (nr_dirty <= (background_thresh + dirty_thresh) / 2)
+               if (nr_dirty <= dirty_freerun_ceiling(dirty_thresh,
+                                                     background_thresh))
                        break;
 
                bdi_thresh = bdi_dirty_limit(bdi, dirty_thresh);

@@ -721,8 +923,9 @@ static void balance_dirty_pages(struct a
                if (!bdi->dirty_exceeded)
                        bdi->dirty_exceeded = 1;
 
-               bdi_update_bandwidth(bdi, dirty_thresh, nr_dirty,
-                                    bdi_thresh, bdi_dirty, start_time);
+               bdi_update_bandwidth(bdi, dirty_thresh, background_thresh,
+                                    nr_dirty, bdi_thresh, bdi_dirty,
+                                    start_time);
 
                /* Note: nr_reclaimable denotes nr_dirty + nr_unstable.
                 * Unstable writes are a feature of certain networked

fs/fs-writeback.c        2011-08-26 15:57:20.000000000 +0800 renamed from fs/fs-writeback.c   2011-08-26 15:57:18.000000000 +0800 +1 -1

@@ -675,7 +675,7 @@ static inline bool over_bground_thresh(v
  static void wb_update_bandwidth(struct bdi_writeback *wb,
                                unsigned long start_time)
  {
-       __bdi_update_bandwidth(wb->bdi, 0, 0, 0, 0, start_time);
+       __bdi_update_bandwidth(wb->bdi, 0, 0, 0, 0, 0, start_time);
  }
 
  /*

include/linux/writeback.h        2011-08-26 15:57:20.000000000 +0800 renamed from include/linux/writeback.h   2011-08-26 15:57:18.000000000 +0800 +1 -1

@@ -141,6 +141,7 @@ unsigned long bdi_dirty_limit(struct bac
 
  void __bdi_update_bandwidth(struct backing_dev_info *bdi,
                            unsigned long thresh,
+                           unsigned long bg_thresh,
                            unsigned long dirty,
                            unsigned long bdi_thresh,
                            unsigned long bdi_dirty,
 
 
--

To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

/*
+ * After a task dirtied this many pages, balance_dirty_pages_ratelimited_nr()
+ * will look to see if it needs to start dirty throttling.
+ *
+ * If dirty_poll_interval is too low, big NUMA machines will call the expensive
+ * global_page_state() too often. So scale it near-sqrt to the safety margin
+ * (the number of pages we may dirty without exceeding the dirty limits).
+ */

Quoted 10 lines · Wu Fengguang

/*
+ * After a task dirtied this many pages, balance_dirty_pages_ratelimited_nr()
+ * will look to see if it needs to start dirty throttling.
+ *
+ * If dirty_poll_interval is too low, big NUMA machines will call the expensive
+ * global_page_state() too often. So scale it near-sqrt to the safety margin
+ * (the number of pages we may dirty without exceeding the dirty limits).
+ */

mm/page-writeback.c +2 -0

+static unsigned long dirty_poll_interval(unsigned long dirty,
+                                        unsigned long thresh)

mm/page-writeback.c static void bdi_update_bandwidth(struct · line 987

Quoted 6 lines

mm/page-writeback.c +6 -0

+{
+       if (thresh > dirty)
+               return 1UL << (ilog2(thresh - dirty) >> 1);
+
+       return 1;
+}

Where does that sqrt come from?

Quoted 11 lines · Peter Zijlstra

Quoted 10 lines · Wu Fengguang

/*
+ * After a task dirtied this many pages, balance_dirty_pages_ratelimited_nr()
+ * will look to see if it needs to start dirty throttling.
+ *
+ * If dirty_poll_interval is too low, big NUMA machines will call the expensive
+ * global_page_state() too often. So scale it near-sqrt to the safety margin
+ * (the number of pages we may dirty without exceeding the dirty limits).
+ */

mm/page-writeback.c +2 -0

+static unsigned long dirty_poll_interval(unsigned long dirty,
+                                        unsigned long thresh)

Quoted 8 lines

mm/page-writeback.c static void bdi_update_bandwidth(struct · line 987

Quoted 6 lines

mm/page-writeback.c +6 -0

+{
+       if (thresh > dirty)
+               return 1UL << (ilog2(thresh - dirty) >> 1);
+
+       return 1;
+}

Where does that sqrt come from?

He does 2^{log_2(x)/2} which, if done in real numbers arithmetics, would
result in x^{1/2}. Given the integer arithmetics, it might be twice as small but still it's some approximation...

Quoted 11 lines · Peter Zijlstra

Quoted 10 lines · Wu Fengguang

/*
+ * After a task dirtied this many pages, balance_dirty_pages_ratelimited_nr()
+ * will look to see if it needs to start dirty throttling.
+ *
+ * If dirty_poll_interval is too low, big NUMA machines will call the expensive
+ * global_page_state() too often. So scale it near-sqrt to the safety margin
+ * (the number of pages we may dirty without exceeding the dirty limits).
+ */

mm/page-writeback.c +2 -0

+static unsigned long dirty_poll_interval(unsigned long dirty,
+                                        unsigned long thresh)

Quoted 8 lines

mm/page-writeback.c static void bdi_update_bandwidth(struct · line 987

Quoted 6 lines

mm/page-writeback.c +6 -0

+{
+       if (thresh > dirty)
+               return 1UL << (ilog2(thresh - dirty) >> 1);
+
+       return 1;
+}

Where does that sqrt come from?

He does 2^{log_2(x)/2} which, if done in real numbers arithmetics, would
result in x^{1/2}. Given the integer arithmetics, it might be twice as small but still it's some approximation...

Quoted 10 lines · Wu Fengguang

/*
+ * After a task dirtied this many pages, balance_dirty_pages_ratelimited_nr()
+ * will look to see if it needs to start dirty throttling.
+ *
+ * If dirty_poll_interval is too low, big NUMA machines will call the expensive
+ * global_page_state() too often. So scale it near-sqrt to the safety margin
+ * (the number of pages we may dirty without exceeding the dirty limits).
+ */

mm/page-writeback.c +2 -0

+static unsigned long dirty_poll_interval(unsigned long dirty,
+                                        unsigned long thresh)

mm/page-writeback.c static void bdi_update_bandwidth(struct · line 987

Quoted 6 lines

mm/page-writeback.c +6 -0

+{
+       if (thresh > dirty)
+               return 1UL << (ilog2(thresh - dirty) >> 1);
+
+       return 1;
+}

Where does that sqrt come from?

So the sqrt naturally leads to less overheads and more N tolerance for large memory servers, which have large (thresh-freerun) gaps.

Quoted 2 lines · Wu Fengguang

So the sqrt naturally leads to less overheads and more N tolerance for large memory servers, which have large (thresh-freerun) gaps.

Thanks, and as you say its an initial guess, later refined using patch 14.

Review

Quoted 6 lines · Wu Fengguang

(quoted diff) +0 -6

-static inline void task_dirties_fraction(struct task_struct *tsk,
-               long *numerator, long *denominator)
-{
-       prop_fraction_single(&vm_dirties, &tsk->dirties,
-                               numerator, denominator);
-} 

it looks like this patch removes all users of tsk->dirties, but doesn't in fact remove the data member from task_struct.

plain text document attachment (bdi-reserve-area)
Keep a minimal pool of dirty pages for each bdi, so that the disk IO queues won't underrun.

It's particularly useful for JBOD and small memory system.

Note that this is not enough when memory is really tight (in comparison to write bandwidth). It may result in (pos_ratio > 1) at the setpoint and push the dirty pages high. This is more or less intended because the bdi is in the danger of IO queue underflow. However the global dirty pages, when pushed close to limit, will eventually conteract our desire to push up the low bdi_dirty.

In low memory JBOD tests we do see disks under-utilized from time to time. The additional fix may be to add a BDI_async_underrun flag to indicate that the block write queue is running low and it's time to quickly fill the queue by unthrottling the tasks regardless of the global limit.

Signed-off-by Wu Fengguang <[email protected]>

---
 mm/page-writeback.c |   26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

mm/page-writeback.c +0 -0

@@ -487,6 +487,16 @@ unsigned long bdi_dirty_limit(struct bac
  *   0 +------------.------------------.----------------------*------------->
  *           freerun^          setpoint^                 limit^   dirty pages

mm/page-writeback.c +0 -0

  * (o) bdi control lines
  *
  * The control lines for the global/bdi setpoints both stretch up to @limit.

@@ -634,6 +644,22 @@ static unsigned long bdi_position_ratio(
 	pos_ratio *= x_intercept - bdi_dirty;
 	do_div(pos_ratio, x_intercept - bdi_setpoint + 1);

Quoted 34 lines · Wu Fengguang

plain text document attachment (bdi-reserve-area)
Keep a minimal pool of dirty pages for each bdi, so that the disk IO queues won't underrun.

It's particularly useful for JBOD and small memory system.

Note that this is not enough when memory is really tight (in comparison to write bandwidth). It may result in (pos_ratio > 1) at the setpoint and push the dirty pages high. This is more or less intended because the bdi is in the danger of IO queue underflow. However the global dirty pages, when pushed close to limit, will eventually conteract our desire to push up the low bdi_dirty.

In low memory JBOD tests we do see disks under-utilized from time to time. The additional fix may be to add a BDI_async_underrun flag to indicate that the block write queue is running low and it's time to quickly fill the queue by unthrottling the tasks regardless of the global limit.

Signed-off-by Wu Fengguang <[email protected]>

---
 mm/page-writeback.c |   26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

mm/page-writeback.c +4 -0

@@ -487,6 +487,16 @@ unsigned long bdi_dirty_limit(struct bac
  *   0 +------------.------------------.----------------------*------------->
  *           freerun^          setpoint^                 limit^   dirty pages
  *
+ * (o) bdi reserve area
+ *
+ * The bdi reserve area tries to keep a reasonable number of dirty pages for
+ * preventing block queue underrun.

mm/page-writeback.c unsigned long bdi_dirty_limit(struct bac · line 498

Quoted 5 lines

mm/page-writeback.c +5 -0

+ *
+ * reserve area, scale up rate as dirty pages drop low
+ * |<----------------------------------------------->|
+ * |-------------------------------------------------------*-------|----------
+ * 0                                           bdi setpoint^       ^bdi_thresh

So why not call the thing bdi freerun ?

Quoted 7 lines

mm/page-writeback.c +0 -0

  * (o) bdi control lines
  *
  * The control lines for the global/bdi setpoints both stretch up to @limit.

@@ -634,6 +644,22 @@ static unsigned long bdi_position_ratio(
 	pos_ratio *= x_intercept - bdi_dirty;
 	do_div(pos_ratio, x_intercept - bdi_setpoint + 1);
 

mm/page-writeback.c static unsigned long bdi_position_ratio( · line 655

Quoted 9 lines

mm/page-writeback.c +9 -0

+	/*
+	 * bdi reserve area, safeguard against dirty pool underrun and disk idle
+	 *
+	 * It may push the desired control point of global dirty pages higher
+	 * than setpoint. It's not necessary in single-bdi case because a
+	 * minimal pool of @freerun dirty pages will already be guaranteed.
+	 */
+	x_intercept = min(write_bw, freerun);
+	if (bdi_dirty < x_intercept) {

So the point of the freerun point is that we never throttle before it, so basically all the below shouldn't be needed at all, right?

mm/page-writeback.c static unsigned long bdi_position_ratio( · line 664

Quoted 9 lines

mm/page-writeback.c +7 -0

+		if (bdi_dirty > x_intercept / 8) {
+			pos_ratio *= x_intercept;
+			do_div(pos_ratio, bdi_dirty);
+		} else
+			pos_ratio *= 8;
+	}
+
 	return pos_ratio;
 }

So why not add:

	if (likely(dirty < freerun))
		return 2;

at the start of this function and leave it at that?

Quoted 5 lines

mm/page-writeback.c static unsigned long bdi_position_ratio( · line 655

Quoted 2 lines

mm/page-writeback.c +2 -0

+   x_intercept = min(write_bw, freerun);
+   if (bdi_dirty < x_intercept) {

So the point of the freerun point is that we never throttle before it, so basically all the below shouldn't be needed at all, right?

Yes!

Quoted 9 lines

Quoted 9 lines

mm/page-writeback.c +7 -0

+           if (bdi_dirty > x_intercept / 8) {
+                   pos_ratio *= x_intercept;
+                   do_div(pos_ratio, bdi_dirty);
+           } else
+                   pos_ratio *= 8;
+   }
+
    return pos_ratio;
 }

Quoted 6 lines

So why not add:

if (likely(dirty < freerun))
return 2;

Quoted 1 line

at the start of this function and leave it at that?

Because we already has

if (nr_dirty < freerun)
break;

in the main balance_dirty_pages() loop ;)

Quoted 17 lines · Wu Fengguang

Quoted 5 lines

Review

Quoted 2 lines

(quoted diff) +2 -0

+   x_intercept = min(write_bw, freerun);
+   if (bdi_dirty < x_intercept) {

So the point of the freerun point is that we never throttle before it, so basically all the below shouldn't be needed at all, right?

Yes!

Quoted 9 lines

Quoted 9 lines

(quoted diff) +7 -0

+           if (bdi_dirty > x_intercept / 8) {
+                   pos_ratio *= x_intercept;
+                   do_div(pos_ratio, bdi_dirty);
+           } else
+                   pos_ratio *= 8;
+   }
+
    return pos_ratio;
 }

Does that mean we can remove this whole block?

Quoted 19 lines · Peter Zijlstra

Quoted 17 lines · Wu Fengguang

Quoted 5 lines

Review

Quoted 2 lines

(quoted diff) +2 -0

+   x_intercept = min(write_bw, freerun);
+   if (bdi_dirty < x_intercept) {

So the point of the freerun point is that we never throttle before it, so basically all the below shouldn't be needed at all, right?

Yes!

Quoted 9 lines

Quoted 9 lines

(quoted diff) +7 -0

+           if (bdi_dirty > x_intercept / 8) {
+                   pos_ratio *= x_intercept;
+                   do_div(pos_ratio, bdi_dirty);
+           } else
+                   pos_ratio *= 8;
+   }
+
    return pos_ratio;
 }

Quoted 1 line

Does that mean we can remove this whole block?

Right, if the bdi freerun concept is proved to work fine.

Unfortunately I find it mostly yields lower performance than bdi reserve area. Patch is attached. If you would like me try other patches, I can easily kick off new tests and redo the comparison.

Here is the nr_written numbers over various JBOD test cases, the larger, the better:

bdi-reserve bdi-freerun diff case
---------------------------------------------------------------------------------------

38375271        31553807      -17.8%	JBOD-10HDD-6G/xfs-100dd-1M-16p-5895M-20
30478879        28631491       -6.1%	JBOD-10HDD-6G/xfs-10dd-1M-16p-5895M-20
29735407        28871956       -2.9%	JBOD-10HDD-6G/xfs-1dd-1M-16p-5895M-20
30850350        28344165       -8.1%	JBOD-10HDD-6G/xfs-2dd-1M-16p-5895M-20
17706200        16174684       -8.6%	JBOD-10HDD-thresh=100M/xfs-100dd-1M-16p-5895M-100M
23374918        14376942      -38.5%	JBOD-10HDD-thresh=100M/xfs-10dd-1M-16p-5895M-100M
20659278        19640375       -4.9%	JBOD-10HDD-thresh=100M/xfs-1dd-1M-16p-5895M-100M
22517497        14552321      -35.4%	JBOD-10HDD-thresh=100M/xfs-2dd-1M-16p-5895M-100M
68287850        61078553      -10.6%	JBOD-10HDD-thresh=2G/xfs-100dd-1M-16p-5895M-2048M
33835247        32018425       -5.4%	JBOD-10HDD-thresh=2G/xfs-10dd-1M-16p-5895M-2048M
30187817        29942083       -0.8%	JBOD-10HDD-thresh=2G/xfs-1dd-1M-16p-5895M-2048M

30563144        30204022       -1.2%	JBOD-10HDD-thresh=2G/xfs-2dd-1M-16p-5895M-2048M
34476862        34645398       +0.5%	JBOD-10HDD-thresh=4G/xfs-10dd-1M-16p-5895M-4096M
30326479        30097263       -0.8%	JBOD-10HDD-thresh=4G/xfs-1dd-1M-16p-5895M-4096M
30446767        30339683       -0.4%	JBOD-10HDD-thresh=4G/xfs-2dd-1M-16p-5895M-4096M
40793956        45936678      +12.6%	JBOD-10HDD-thresh=800M/xfs-100dd-1M-16p-5895M-800M
27481305        24867282       -9.5%	JBOD-10HDD-thresh=800M/xfs-10dd-1M-16p-5895M-800M
25651257        22507406      -12.3%	JBOD-10HDD-thresh=800M/xfs-1dd-1M-16p-5895M-800M
19849350        21298787       +7.3%	JBOD-10HDD-thresh=800M/xfs-2dd-1M-16p-5895M-800M

BTW, I also compared the IO-less patchset and the vanilla kernel's JBOD performance. Basically, the performance is lightly improved under large memory, and reduced a lot in small memory servers.

vanillla IO-less
--------------------------------------------------------------------------------

 26508063 17706200      -33.2%  JBOD-10HDD-thresh=100M/xfs-100dd-1M-16p-5895M-100M
 23767810 23374918       -1.7%  JBOD-10HDD-thresh=100M/xfs-10dd-1M-16p-5895M-100M
 28032891 20659278      -26.3%  JBOD-10HDD-thresh=100M/xfs-1dd-1M-16p-5895M-100M
 26049973 22517497      -13.6%  JBOD-10HDD-thresh=100M/xfs-2dd-1M-16p-5895M-100M

There are still some itches in JBOD..

Quoted 6 lines

BTW, I also compared the IO-less patchset and the vanilla kernel's JBOD performance. Basically, the performance is lightly improved under large memory, and reduced a lot in small memory servers.

vanillla IO-less
--------------------------------------------------------------------------------

[...]

Quoted 6 lines

 26508063 17706200      -33.2%  JBOD-10HDD-thresh=100M/xfs-100dd-1M-16p-5895M-100M
 23767810 23374918       -1.7%  JBOD-10HDD-thresh=100M/xfs-10dd-1M-16p-5895M-100M
 28032891 20659278      -26.3%  JBOD-10HDD-thresh=100M/xfs-1dd-1M-16p-5895M-100M
 26049973 22517497      -13.6%  JBOD-10HDD-thresh=100M/xfs-2dd-1M-16p-5895M-100M

There are still some itches in JBOD..

OK, in the dirty_bytes=100M case, I find that the bdi threshold _and_ writeout bandwidth may drop close to 0 in long periods. This change may avoid one bdi being stuck:

/*
* bdi reserve area, safeguard against dirty pool underrun and disk idle *
* It may push the desired control point of global dirty pages higher
* than setpoint. It's not necessary in single-bdi case because a
* minimal pool of @freerun dirty pages will already be guaranteed. */

After lots of experiments, I end up with this bdi reserve point

+ x_intercept = bdi_thresh / 2 + MIN_WRITEBACK_PAGES;

together with this chunk to avoid a bdi stuck in bdi_thresh=0 state:

The above changes are good enough to keep reasonable amount of bdi dirty pages, so the bdi underrun flag ("[PATCH 11/18] block: add bdi flag to indicate risk of io queue underrun") is dropped.

I also tried various bdi freerun patches, however the results are not satisfactory. Basically the bdi reserve area approach (this patch) yields noticeably more smooth/resilient behavior than the freerun/underrun approaches. I noticed that the bdi underrun flag could lead to sudden surge of dirty pages (especially if not safeguarded by the dirty_exceeded condition) in the very small window..

On Wed, 2011-09-28 at 22:02 +0800, Wu Fengguang wrote:

/me attempts to swap back neurons related to writeback

Quoted 6 lines

After lots of experiments, I end up with this bdi reserve point

+ x_intercept = bdi_thresh / 2 + MIN_WRITEBACK_PAGES;

together with this chunk to avoid a bdi stuck in bdi_thresh=0 state:

Review static unsigned long bdi_position_ratio( · line 596

Quoted 8 lines

(quoted diff) +1 -0

@@ -590,6 +590,7 @@ static unsigned long bdi_position_ratio(
         */
        if (unlikely(bdi_thresh > thresh))
                bdi_thresh = thresh;
+       bdi_thresh = max(bdi_thresh, (limit - dirty) / 8);
        /*
         * scale global setpoint to bdi's:
         *      bdi_setpoint = setpoint * bdi_thresh / thresh

So you cap bdi_thresh at a minimum of (limit-dirty)/8 which can be pretty close to 0 if we have a spike in dirty or a negative spike in writeout bandwidth (sudden seeks or whatnot).

Quoted 4 lines

The above changes are good enough to keep reasonable amount of bdi dirty pages, so the bdi underrun flag ("[PATCH 11/18] block: add bdi flag to indicate risk of io queue underrun") is dropped.

That sounds like goodness ;-)

Quoted 7 lines

I also tried various bdi freerun patches, however the results are not satisfactory. Basically the bdi reserve area approach (this patch) yields noticeably more smooth/resilient behavior than the freerun/underrun approaches. I noticed that the bdi underrun flag could lead to sudden surge of dirty pages (especially if not safeguarded by the dirty_exceeded condition) in the very small window..

OK, so let me try and parse this magic:

(quoted diff) +8 -0

+       x_intercept = bdi_thresh / 2 + MIN_WRITEBACK_PAGES;
+       if (bdi_dirty < x_intercept) {
+               if (bdi_dirty > x_intercept / 8) {
+                       pos_ratio *= x_intercept;
+                       do_div(pos_ratio, bdi_dirty);
+               } else
+                       pos_ratio *= 8;
+       }

So we set our target some place north of MIN_WRITEBACK_PAGES: if we're short we add a factor of: x_intercept/bdi_dirty.

Now, since bdi_dirty < x_intercept, this is > 1 and thus we promote more dirties.

Additionally we don't let the factor get larger than 8 to avoid silly large fluctuations (8 already seems quite generous to me).

Now I guess the only problem is when nr_bdi * MIN_WRITEBACK_PAGES ~ limit, at which point things go pear shaped.

Quoted 2 lines

Now I guess the only problem is when nr_bdi * MIN_WRITEBACK_PAGES ~ limit, at which point things go pear shaped.

Yes. In that case the global @dirty will always be drove up to @limit. Once @dirty dropped reasonably below, whichever bdi task wakeup first will take the chance to fill the gap, which is not fair for bdi's of different speed.

Let me retry the thresh=1M,10M test cases without MIN_WRITEBACK_PAGES. Hopefully the removal of it won't impact performance a lot.

Quoted 8 lines · Wu Fengguang

Quoted 2 lines

Now I guess the only problem is when nr_bdi * MIN_WRITEBACK_PAGES ~ limit, at which point things go pear shaped.

Yes. In that case the global @dirty will always be drove up to @limit. Once @dirty dropped reasonably below, whichever bdi task wakeup first will take the chance to fill the gap, which is not fair for bdi's of different speed.

Quoted 2 lines

Let me retry the thresh=1M,10M test cases without MIN_WRITEBACK_PAGES. Hopefully the removal of it won't impact performance a lot.

Right, so alternatively we could try an argument that this is sufficiently rare and shouldn't happen. People with lots of disks tend to also have lots of memory, etc.

If we do find it happens we can always look at it again.

Hi Peter,

Quoted 25 lines · Wu Fengguang

Quoted 6 lines

BTW, I also compared the IO-less patchset and the vanilla kernel's JBOD performance. Basically, the performance is lightly improved under large memory, and reduced a lot in small memory servers.

vanillla IO-less
--------------------------------------------------------------------------------

[...]

Quoted 6 lines

 26508063 17706200      -33.2%  JBOD-10HDD-thresh=100M/xfs-100dd-1M-16p-5895M-100M
 23767810 23374918       -1.7%  JBOD-10HDD-thresh=100M/xfs-10dd-1M-16p-5895M-100M
 28032891 20659278      -26.3%  JBOD-10HDD-thresh=100M/xfs-1dd-1M-16p-5895M-100M
 26049973 22517497      -13.6%  JBOD-10HDD-thresh=100M/xfs-2dd-1M-16p-5895M-100M

There are still some itches in JBOD..

OK, in the dirty_bytes=100M case, I find that the bdi threshold _and_ writeout bandwidth may drop close to 0 in long periods. This change may avoid one bdi being stuck:

/*
* bdi reserve area, safeguard against dirty pool underrun and disk idle *
* It may push the desired control point of global dirty pages higher
* than setpoint. It's not necessary in single-bdi case because a
* minimal pool of @freerun dirty pages will already be guaranteed. */

Review

Quoted 2 lines

(quoted diff) +1 -1

-       x_intercept = min(write_bw, freerun);
+       x_intercept = min(write_bw + MIN_WRITEBACK_PAGES, freerun);

After lots of experiments, I end up with this bdi reserve point

+ x_intercept = bdi_thresh / 2 + MIN_WRITEBACK_PAGES;

together with this chunk to avoid a bdi stuck in bdi_thresh=0 state:

(quoted diff) +1 -0

@@ -590,6 +590,7 @@ static unsigned long bdi_position_ratio(
         */
        if (unlikely(bdi_thresh > thresh))
                bdi_thresh = thresh;
+       bdi_thresh = max(bdi_thresh, (limit - dirty) / 8);
        /*
         * scale global setpoint to bdi's:
         *      bdi_setpoint = setpoint * bdi_thresh / thresh

The above changes are good enough to keep reasonable amount of bdi dirty pages, so the bdi underrun flag ("[PATCH 11/18] block: add bdi flag to indicate risk of io queue underrun") is dropped.

I also tried various bdi freerun patches, however the results are not satisfactory. Basically the bdi reserve area approach (this patch) yields noticeably more smooth/resilient behavior than the freerun/underrun approaches. I noticed that the bdi underrun flag could lead to sudden surge of dirty pages (especially if not safeguarded by the dirty_exceeded condition) in the very small window..

To dig performance increases/drops out of the large number of test results, I wrote a convenient script (attached) to compare the vmstat:nr_written numbers between 2+ set of test runs. It helped a lot for fine tuning the parameters for different cases.

The current JBOD performance numbers are encouraging:

$ ./compare.rb JBOD*/*-vanilla+ JBOD*/*-bgthresh3+
      3.1.0-rc4-vanilla+      3.1.0-rc4-bgthresh3+
------------------------  ------------------------
                52934365        +3.2%     54643527  JBOD-10HDD-thresh=100M/ext4-100dd-1M-24p-16384M-100M:10-X
                45488896       +18.2%     53785605  JBOD-10HDD-thresh=100M/ext4-10dd-1M-24p-16384M-100M:10-X
                47217534       +12.2%     53001031  JBOD-10HDD-thresh=100M/ext4-1dd-1M-24p-16384M-100M:10-X
                32286924       +25.4%     40492312  JBOD-10HDD-thresh=100M/xfs-10dd-1M-24p-16384M-100M:10-X
                38676965       +14.2%     44177606  JBOD-10HDD-thresh=100M/xfs-1dd-1M-24p-16384M-100M:10-X
                59662173       +11.1%     66269621  JBOD-10HDD-thresh=800M/ext4-10dd-1M-24p-16384M-800M:10-X
                57510438        +2.3%     58855181  JBOD-10HDD-thresh=800M/ext4-1dd-1M-24p-16384M-800M:10-X
                63691922       +64.0%    104460352  JBOD-10HDD-thresh=800M/xfs-100dd-1M-24p-16384M-800M:10-X
                51978567       +16.0%     60298210  JBOD-10HDD-thresh=800M/xfs-10dd-1M-24p-16384M-800M:10-X
                47641062        +6.4%     50681038  JBOD-10HDD-thresh=800M/xfs-1dd-1M-24p-16384M-800M:10-X

mm/page-writeback.c static void balance_dirty_pages(struct a · line 1075

Quoted 11 lines · Wu Fengguang

+++ linux-next/mm/page-writeback.c 2011-08-31 14:40:58.000000000 +0800

mm/page-writeback.c +3 -0

@@ -1067,6 +1067,9 @@ static void balance_dirty_pages(struct a
                                     nr_dirty, bdi_thresh, bdi_dirty,
                                     start_time);
 
+               if (unlikely(!dirty_exceeded && bdi_async_underrun(bdi)))
+                       break;
+
                dirty_ratelimit = bdi->dirty_ratelimit;
                pos_ratio = bdi_position_ratio(bdi, dirty_thresh,
                                               background_thresh, nr_dirty,

So dirty_exceeded looks like:

1109 dirty_exceeded = (bdi_dirty > bdi_thresh) ||
1110 (nr_dirty > dirty_thresh);

Would it make sense to write it as:

	if (nr_dirty > dirty_thresh || 
	    (nr_dirty > freerun && bdi_dirty > bdi_thresh))
		dirty_exceeded = 1;

So that we don't actually throttle bdi thingies when we're still in the freerun area?

Quoted 25 lines · Peter Zijlstra

mm/page-writeback.c static void balance_dirty_pages(struct a · line 1075

Quoted 11 lines · Wu Fengguang

+++ linux-next/mm/page-writeback.c 2011-08-31 14:40:58.000000000 +0800

mm/page-writeback.c +3 -0

@@ -1067,6 +1067,9 @@ static void balance_dirty_pages(struct a
                                     nr_dirty, bdi_thresh, bdi_dirty,
                                     start_time);
 
+               if (unlikely(!dirty_exceeded && bdi_async_underrun(bdi)))
+                       break;
+
                dirty_ratelimit = bdi->dirty_ratelimit;
                pos_ratio = bdi_position_ratio(bdi, dirty_thresh,
                                               background_thresh, nr_dirty,

So dirty_exceeded looks like:

1109 dirty_exceeded = (bdi_dirty > bdi_thresh) ||
1110 (nr_dirty > dirty_thresh);

Would it make sense to write it as:

	if (nr_dirty > dirty_thresh || 
	    (nr_dirty > freerun && bdi_dirty > bdi_thresh))
		dirty_exceeded = 1;

Quoted 2 lines

So that we don't actually throttle bdi thingies when we're still in the freerun area?

Sounds not necessary -- (nr_dirty > freerun) is implicitly true because there is a big break early in the loop:

if (nr_dirty > freerun)
break;

mm/page-writeback.c +8 -0

+	/*
+	 * Scale up pause time for concurrent dirtiers in order to reduce CPU
+	 * overheads.
+	 *
+	 * (N * 20ms) on 2^N concurrent tasks.
+	 */
+	if (hi > lo)
+		t += (hi - lo) * (20 * HZ) / 1024;

+}

Review

Quoted 9 lines · Wu Fengguang

(quoted diff) +9 -0

+static unsigned long bdi_max_pause(struct backing_dev_info *bdi,
+				   unsigned long bdi_dirty)
+{
+	unsigned long hi = ilog2(bdi->write_bandwidth);
+	unsigned long lo = ilog2(bdi->dirty_ratelimit);
+	unsigned long t;
+
+	/* target for ~10ms pause on 1-dd case */
+	t = HZ / 50;

1k/50 usually ends up being 20 something

Quoted 13 lines

(quoted diff) +13 -0

+	/*
+	 * Scale up pause time for concurrent dirtiers in order to reduce CPU
+	 * overheads.
+	 *
+	 * (N * 20ms) on 2^N concurrent tasks.
+	 */
+	if (hi > lo)
+		t += (hi - lo) * (20 * HZ) / 1024;
+
+	/*
+	 * Limit pause time for small memory systems. If sleeping for too long
+	 * time, a small pool of dirty/writeback pages may go empty and disk go
+	 * idle.

Review

Quoted 5 lines

(quoted diff) +5 -0

+	 *
+	 * 1ms for every 1MB; may further consider bdi bandwidth.
+	 */
+	if (bdi_dirty)
+		t = min(t, bdi_dirty >> (30 - PAGE_CACHE_SHIFT - ilog2(HZ)));

Yeah, I would add the bdi->avg_write_bandwidth term in there, 1g/s as an avg bandwidth is just too wrong..

Review

Quoted 6 lines

(quoted diff) +6 -0

+
+	/*
+	 * The pause time will be settled within range (max_pause/4, max_pause).
+	 * Apply a minimal value of 4 to get a non-zero max_pause/4.
+	 */
+	return clamp_val(t, 4, MAX_PAUSE);

So you limit to 50ms min? That still seems fairly large. Is that because your min sleep granularity might be something like 10ms since you're using jiffies?

So yeah, the HZ value does impact the minimal available sleep time...

Quoted 1 line · Wu Fengguang

So yeah, the HZ value does impact the minimal available sleep time...

There's always schedule_hrtimeout() and we could trivially add a io_schedule_hrtimeout() variant if you need it.

plain text document attachment (max-pause-adaption) The dirty pause time shall ultimately be controlled by adjusting nr_dirtied_pause, since there is relationship

pause = pages_dirtied / task_ratelimit

Assuming

pages_dirtied ~= nr_dirtied_pause
task_ratelimit ~= dirty_ratelimit

We get

nr_dirtied_pause ~= dirty_ratelimit * desired_pause

Here dirty_ratelimit is preferred over task_ratelimit because it's more stable.

It's also important to limit possible large transitional errors:

- bw is changing quickly
- pages_dirtied << nr_dirtied_pause on entering dirty exceeded area
- pages_dirtied >> nr_dirtied_pause on btrfs (to be improved by a separate fix, but still expect non-trivial errors)

So we end up using the above formula inside clamp_val().

The best test case for this code is to run 100 "dd bs=4M" tasks on btrfs and check its pause time distribution.

Signed-off-by Wu Fengguang <[email protected]>

---
 mm/page-writeback.c |   15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

mm/page-writeback.c +5 -1

@@ -1193,7 +1193,20 @@ pause:
 	if (!dirty_exceeded && bdi->dirty_exceeded)
 		bdi->dirty_exceeded = 0;
 
-	current->nr_dirtied_pause = dirty_poll_interval(nr_dirty, dirty_thresh);
+	if (pause == 0)
+		current->nr_dirtied_pause =
+				dirty_poll_interval(nr_dirty, dirty_thresh);
+	else if (period <= max_pause / 4 &&
+		 pages_dirtied >= current->nr_dirtied_pause)

Quoted 54 lines · Wu Fengguang

plain text document attachment (max-pause-adaption) The dirty pause time shall ultimately be controlled by adjusting nr_dirtied_pause, since there is relationship

pause = pages_dirtied / task_ratelimit

Assuming

pages_dirtied ~= nr_dirtied_pause
task_ratelimit ~= dirty_ratelimit

We get

nr_dirtied_pause ~= dirty_ratelimit * desired_pause

Here dirty_ratelimit is preferred over task_ratelimit because it's more stable.

It's also important to limit possible large transitional errors:

- bw is changing quickly
- pages_dirtied << nr_dirtied_pause on entering dirty exceeded area
- pages_dirtied >> nr_dirtied_pause on btrfs (to be improved by a separate fix, but still expect non-trivial errors)

So we end up using the above formula inside clamp_val().

The best test case for this code is to run 100 "dd bs=4M" tasks on btrfs and check its pause time distribution.

Signed-off-by Wu Fengguang <[email protected]>

---
 mm/page-writeback.c |   15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

mm/page-writeback.c +10 -1

@@ -1193,7 +1193,20 @@ pause:
 	if (!dirty_exceeded && bdi->dirty_exceeded)
 		bdi->dirty_exceeded = 0;
 
-	current->nr_dirtied_pause = dirty_poll_interval(nr_dirty, dirty_thresh);
+	if (pause == 0)
+		current->nr_dirtied_pause =
+				dirty_poll_interval(nr_dirty, dirty_thresh);
+	else if (period <= max_pause / 4 &&
+		 pages_dirtied >= current->nr_dirtied_pause)
+		current->nr_dirtied_pause = clamp_val(
+					dirty_ratelimit * (max_pause / 2) / HZ,
+					pages_dirtied + pages_dirtied / 8,
+					pages_dirtied * 4);
+	else if (pause >= max_pause)