Zola2026-03-01T00:00:00+00:00https://dimtion.fr/atom.xmlLogit soft-capping2026-03-01T00:00:00+00:002026-03-01T00:00:00+00:00
Unknown
https://dimtion.fr/wiki/ml/soft-capping/<h2 id="logit">Logit</h2>
<p>From <a rel="external" href="https://developers.google.com/machine-learning/glossary/#logits">google</a>:</p>
<blockquote>
<p>the vector of raw (non-normalized) predictions that a classification model generates, which is ordinarily then passed to a normalization function. If the model is solving a multi-class classification problem, logits typically become an input to the softmax function. The softmax function then generates a vector of (normalized) probabilities with one value for each possible class.</p>
</blockquote>
<h2 id="logit-soft-capping">Logit soft-capping</h2>
<p>Logit softcapping (introduced in <a rel="external" href="https://storage.googleapis.com/deepmind-media/gemma/gemma-2-report.pdf">Gemma 2</a>) is a technique to cap the value between $-soft\_cap$ and $+soft\_cap$:</p>
<p>$$
logits ← soft\_cap \cdot tanh(\frac{logits}{soft\_cap})
$$</p>
<div >
<style>
#cap-slider::-webkit-slider-thumb {
-webkit-appearance: none;
width: 13px; height: 13px;
border-radius: 50%;
background: var(--link-color);
cursor: pointer;
}
#softcap-chart .grid line {
stroke: var(--font-color);
stroke-opacity: 0.1;
}
#softcap-chart .axis path, #softcap-chart .axis line {
stroke: var(--font-color);
opacity: 0.3;
stroke-opacity: 0.3;
}
#softcap-chart .axis text {
fill: var(--font-color);
opacity: 0.8;
font-size: 11px;
font-family: monospace;
}
</style>
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/7.8.5/d3.min.js"></script>
<div style="display:flex;flex-direction:column;">
<div id="softcap-chart" style="width:100%;"></div>
<div style="display:flex;align-items:center;justify-content:center;gap:12px;margin-top:12px;">
<label for="cap-slider" style="font-size:11px;letter-spacing:0.15em;text-transform:uppercase;color:var(--font-color);opacity:1;">cap</label>
<input type="range" id="cap-slider" min="1" max="60" step="0.5" value="30" style="-webkit-appearance:none;width:220px;height:2px;background:var(--glass-border);outline:none;border-radius:1px;">
<span id="cap-val" style="font-size:13px;color:var(--link-color);min-width:28px;">30</span>
</div>
</div>
<script>
(function() {
// --- Theme ---
function getTheme() {
const cs = getComputedStyle(document.documentElement);
return {
colorLink: cs.getPropertyValue('--link-color').trim() || '#5b9dd9',
colorAlt: cs.getPropertyValue('--title-color-alt').trim() || '#ed8796',
colorFont: cs.getPropertyValue('--font-color').trim() || '#2d3748',
};
}
const W = 520, H = 380;
const margin = { top: 20, right: 20, bottom: 40, left: 50 };
const w = W - margin.left - margin.right;
const h = H - margin.top - margin.bottom;
const svg = d3.select('#softcap-chart').append('svg')
.attr('viewBox', `0 0 ${W} ${H}`)
.attr('width', '100%')
.style('height', 'auto');
const g = svg.append('g')
.attr('transform', `translate(${margin.left},${margin.top})`);
const xRange = 80;
const x = d3.scaleLinear().domain([-xRange, xRange]).range([0, w]);
const y = d3.scaleLinear().domain([-xRange, xRange]).range([h, 0]);
g.append('g').attr('class', 'grid')
.call(d3.axisLeft(y).tickSize(-w).tickFormat('').ticks(8));
g.append('g').attr('class', 'grid')
.attr('transform', `translate(0,${h})`)
.call(d3.axisBottom(x).tickSize(-h).tickFormat('').ticks(8));
g.append('g').attr('class', 'axis').call(d3.axisLeft(y).ticks(8));
g.append('g').attr('class', 'axis').attr('transform', `translate(0,${h})`).call(d3.axisBottom(x).ticks(8));
const xLabel = g.append('text').attr('x', w/2).attr('y', h+36)
.attr('opacity', 1).attr('text-anchor','middle')
.style('font-size','11px').style('font-family','monospace').text('logits (input)');
const yLabel = g.append('text').attr('transform','rotate(-90)').attr('x',-h/2).attr('y',-40)
.attr('opacity', 1).attr('text-anchor','middle')
.style('font-size','11px').style('font-family','monospace').text('output');
const pts = d3.range(-xRange, xRange + 0.2, 0.2);
const line = fn => d3.line().x(d => x(d)).y(d => y(fn(d)));
// Identity
const identityPath = g.append('path').datum(pts).attr('fill','none')
.attr('stroke-opacity', 0.3).attr('stroke-width', 1)
.attr('d', line(d => d));
// Soft-cap
const softPath = g.append('path').attr('fill','none').attr('stroke-width', 2);
// Hard clip
const clipPath = g.append('path').attr('fill','none')
.attr('stroke-width', 2).attr('stroke-dasharray','6,4');
// Indicator lines
const softLinePos = g.append('line').attr('stroke-width',0.5).attr('stroke-dasharray','2,4').attr('opacity',1);
const softLineNeg = g.append('line').attr('stroke-width',0.5).attr('stroke-dasharray','2,4').attr('opacity',1);
// Legend
const legend = g.append('g').attr('transform', `translate(${w - 148}, 8)`);
const legendItems = [
{ dash: null, label: 'soft-cap', key: 'link' },
{ dash: '6,4', label: 'hard clip', key: 'alt' },
{ dash: null, label: 'y=x', key: 'font' },
].map((item, i) => {
const row = legend.append('g').attr('transform', `translate(0, ${i * 20})`);
const swatch = row.append('line').attr('x1',0).attr('x2',22).attr('y1',6).attr('y2',6)
.attr('stroke-width', 2).attr('stroke-opacity', 1)
.attr('stroke-dasharray', item.dash || null);
const label = row.append('text').attr('x',30).attr('y',10)
.attr('opacity', 1).style('font-size','11px').style('font-family','monospace').text(item.label);
return { swatch, label, key: item.key };
});
function update() {
const cap = +document.getElementById('cap-slider').value;
const { colorLink, colorAlt, colorFont } = getTheme();
document.getElementById('cap-val').textContent = cap;
// Update paths
identityPath.attr('stroke', colorFont);
softPath.attr('stroke', colorLink).attr('d', line(d => cap * Math.tanh(d / cap))(pts));
clipPath.attr('stroke', colorAlt).attr('d', line(d => Math.max(-cap, Math.min(cap, d)))(pts));
// Update indicator lines
softLinePos.attr('stroke', colorLink).attr('x1', x(-xRange)).attr('x2', x(xRange)).attr('y1', y(cap)).attr('y2', y(cap));
softLineNeg.attr('stroke', colorLink).attr('x1', x(-xRange)).attr('x2', x(xRange)).attr('y1', y(-cap)).attr('y2', y(-cap));
// Update axis labels
xLabel.attr('fill', colorFont);
yLabel.attr('fill', colorFont);
// Update legend
legendItems.forEach(({ swatch, label, key }) => {
const color = key === 'link' ? colorLink : key === 'alt' ? colorAlt : colorFont;
swatch.attr('stroke', color);
label.attr('fill', colorFont);
});
}
document.getElementById('cap-slider').addEventListener('input', update);
// Re-apply theme colors when the toggle in the navbar changes data-theme
new MutationObserver(update).observe(document.documentElement, {
attributes: true, attributeFilter: ['data-theme']
});
// Also catch OS-level preference changes
if (window.matchMedia) {
window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', update);
}
update();
})();
</script>
</div>
<p>From: <a rel="external" href="https://arxiv.org/pdf/2410.16682">Methods of Improving LLM Training Stability</a></p>
Calcification of Technology2025-12-07T00:00:00+00:002025-12-07T00:00:00+00:00
Unknown
https://dimtion.fr/blog/2025/tech-calcification/<p>One thing to notice when studying the history of tech or the <a rel="external" href="https://en.wikipedia.org/wiki/History_of_the_Internet">internet</a> is the <em>winner takes all</em> trend that results in slowly making tech a higher and higher tower of complexity.</p>
<p>Here, I call tech calcification the observed principle that once a technology is sufficiently widespread, new capabilities of a technology are built on top of it instead of next to it. Here are a few examples:</p>
<ul>
<li>Everything is built on top of <a rel="external" href="https://labs.ripe.net/author/hisham_ibrahim/schrodingers-ipv6-cat/">IPv4</a>.</li>
<li>Everything is built on top of <a rel="external" href="https://samsaffron.com/archive/2015/12/29/websockets-caution-required">TCP</a>.</li>
<li>Everything is built on top of <a rel="external" href="https://developer.mozilla.org/en-US/docs/Web/API/USB">HTTP</a>.</li>
<li>Everything is built on top of a <a rel="external" href="https://www.netflix.com/">client-server</a> architecture.</li>
<li>Everything is built on top of <a rel="external" href="https://www.jsonrpc.org/">JSON</a>.</li>
<li>Everything is built on top of <a rel="external" href="https://webassembly.org/">JavaScript</a>.</li>
<li>Everything is built on top of <a rel="external" href="https://github.com/microsoft/WSL2-Linux-Kernel">Linux</a>.</li>
<li>Everything is built on top of <a rel="external" href="https://en.wikipedia.org/wiki/Carrier-grade_NAT">NATed</a> networks.</li>
<li>Everything is built on top of MCP…</li>
</ul>
<p>When a technology becomes widespread, it naturally gathers enough momentum to attract more people to work with it. The community memetic effect (or network effect [sic]) of the internet is its main driving force. For example, when enough people use a web-browser daily it is only natural that the web becomes the main software distribution platform. It is <em>expected</em> that a simple document browser becomes a high performance <a rel="external" href="https://developer.mozilla.org/en-US/docs/Web/API/WebGPU_API">graphics renderer</a>.</p>
<p>The 80s was a booming period for operating systems, with OSs specialized for micro-computing, mainframe, workstations etc… Today the market is mostly consolidated in 4 main OS (Windows, Linux, MacOS, BSD), and those can be grouped only 2 categories: POSIX & Windows. And now, <a rel="external" href="https://github.com/microsoft/WSL2-Linux-Kernel">Windows is Linux</a> … Whole billion dollar sub-industries today live by specializing in securing, micro-optimizing, and spreading best practices for those 4 OS.</p>
<p>There are other reasons beyond just the community effect. Once a technology is widespread enough, more eyes are present to (1) improve the said technology, (2) do security assessments (3) experiment and spread said <em>best practices</em>. While the path is chaotic, it generally results in technologies that are somewhat <a rel="external" href="https://x.com/jarredsumner/status/1499225725492076544">efficient</a>, very <a rel="external" href="https://www.espruino.com/ESP32">versatile</a>, and <em>mostly</em> <a rel="external" href="https://v8.dev/blog/sandbox">secure</a>. Though, champion of none.</p>
<p>A calcified technology is also very difficult to replace, too many people to convince, too many devices to replace (or keep backward compatibility), too many best practices to re-learn. <a rel="external" href="https://quicwg.org/">QUIC</a> protocol is a good example of a technology that while mostly superior to TCP, has <a rel="external" href="https://daniel.haxx.se/blog/2024/06/10/http-3-in-curl-mid-2024/">difficulties</a> replacing it due to how optimized TCP+TLS implementations are. From <a rel="external" href="https://www.corsix.org/content/galois-field-instructions-2021-cpus">custom CPU instructions</a>, to decade of research poured into testing all the limits of TCP, not to mention <a rel="external" href="https://httptoolkit.com/blog/http3-quic-open-source-support-nowhere/">millions of routers</a>, or high tower <a rel="external" href="https://github.com/nodejs/node/issues/57281">http server stacks</a>.</p>
<p>While I wish the world would move on from some calcified technologies, that there was more OS diversity, more network protocol diversity, less cargo-cult in serialization patterns, I do value that having an <em>imperfect</em> building block that is set in calcium allows the larger community to innovate on top of it.</p>
<hr />
<p style="text-align: end">
<i>And no, AI will not solve this problem</i>
</p>
<!--
## Problems
- efficiency is generally the loser: everything is built on top of a more and more complex stack of things. Everything is slower
- complexity
## Calcification is good actually
- N protocols -> N+1 protocols problem
- eventually even calcified a technology still evolves (e.g. http)
- the reason why a technology becomes big and calcified is due to environmental pressure that alternatives did not consider in time
- e.g. wasm is built on top of the web tech as it is the best app distribution platform: flash, java applet
- e.g. web sockets have better security where blank TCP can't provide
-
## How to fix
Hard question, this is fundamentally social question, some reasons could be addressed by better education, just knowing that a layer exist, would allow engineers to see more of the options to build something new
In each case, technologically superior alternatives exists, but because they never reached a critical adoption point, they never became the de-facto standard.
There are many reasons why this happens, and some are good,
- security
- skills
- new engineers building on top of what they know
- legal compliance
- once something is big, it is hard to introduce variants (e.g quic is a new protocol, and is having hard time to come in to replace TCP where needed)
- cargo cult
-->Optimizing GutterMarks.nvim2025-11-30T00:00:00+00:002025-11-30T00:00:00+00:00
Unknown
https://dimtion.fr/blog/2025/optimizing-guttermarks-nvim/<p>A few weeks ago, I shared GutterMarks.nvim (<a href="/blog/2025/guttermarks-nvim">blog</a>, <a rel="external" href="https://github.com/dimtion/guttermarks.nvim">GitHub</a>) a Neovim plugin to allow displaying the nvim marks in the gutter of a buffer. It got positive feedback, and since then I added a couple of convenient actions, and fixed a couple of minor bugs that the community had found.</p>
<p>One thing I spent more time recently is optimizing the performances of the plugin. In the <a rel="external" href="https://github.com/dimtion/guttermarks.nvim/?tab=readme-ov-file#why-this-plugin">README</a> I mention that the plugin is faster that other implementations (that I found), as I'm avoid using any timers to compute the marks. However, there is always headroom for faster and better. So I set to myself as an exercise to see how much more it could be optimized.</p>
<p>The first step for any optimization is: measure. I did not find any suitable Neovim benchmark suite for plugins. So I wrote a little helper <a rel="external" href="https://github.com/dimtion/guttermarks.nvim/blob/main/test/bench/helpers_bench.lua">benchmark suite</a>. This is really the most basic implementation leveraging <a rel="external" href="https://github.com/nvim-mini/mini.nvim/blob/main/readmes/mini-test.md">Mini.test</a>: Create a reproducible environment, run a test N times, print results, run tests regularly to validate regression.</p>
<p>As this is a hobby project, I did not spent much time to remove the bias from the benchmark suite (warm-up, validate variance, remove constant factors...). If I ever do this, I may then publish this as a proper benchmark suite.</p>
<p>So now we have this common pattern:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="lua"><span class="giallo-l"><span>T[</span><span style="color: #A6D189;">"refresh with 1 local mark"</span><span>] </span><span style="color: #81C8BE;">=</span><span style="color: #CA9EE6;"> function</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> -- Setup a buffer with 100 lines</span></span>
<span class="giallo-l"><span> bench_helpers.</span><span style="color: #8CAAEE;font-style: italic;">setup_local_marks</span><span>(child, </span><span style="color: #EF9F76;">1</span><span>, </span><span style="color: #EF9F76;">100</span><span>)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> -- Run the benchmark, by default 100 times:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> local</span><span> results</span><span style="color: #81C8BE;"> =</span><span> bench_helpers.</span><span style="color: #8CAAEE;font-style: italic;">benchmark</span><span>(</span><span style="color: #A6D189;">"refresh with 1 local mark"</span><span>, </span><span style="color: #CA9EE6;">function</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span> child.</span><span style="color: #8CAAEE;font-style: italic;">lua</span><span>(</span><span style="color: #A6D189;">[[require("guttermarks").refresh()]]</span><span>)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> end</span><span>)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> -- Leverage MiniTest notes to display the test result:</span></span>
<span class="giallo-l"><span> MiniTest.</span><span style="color: #8CAAEE;font-style: italic;">add_note</span><span>(bench_helpers.</span><span style="color: #8CAAEE;font-style: italic;">format_note</span><span>(results))</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;">end</span></span></code></pre>
<p>Which prints:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">NOTE</span><span style="color: #A6D189;"> in test/bench/bench_refresh.lua</span><span style="color: #81C8BE;"> |</span><span style="color: #8CAAEE;font-style: italic;"> refresh</span><span style="color: #A6D189;"> with</span><span style="color: #EF9F76;"> 1</span><span style="color: #A6D189;"> local mark: it=</span><span style="color: #EF9F76;">0100</span></span>
<span class="giallo-l"><span>mean</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">0.036ms</span><span> median</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">0.034ms</span><span> min</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">0.028ms</span><span> max</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">0.067ms</span><span> stddev</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">0.007ms</span></span></code></pre>
<p>While not perfect, having the mean, median, min, max and sdtddev gives us a good idea of the distribution, thus helps us validating our priors and if any optimization is worthwhile.</p>
<blockquote>
<p>Note I'm only optimizing for latency (as opposed to memory or throughput), since for this project I'm working with the assumption that memory usage is low.</p>
</blockquote>
<p>I decided to optimize the critical loop of the plugin: the <code>refresh()</code> function. This function goal is to list all the configured marks, and refresh the gutter with the latest values. In particular I created 2 scenarios:</p>
<ul>
<li><strong>Light Scenario</strong>: 3 marks in a 500 lines file, call refresh 1000 times</li>
<li><strong>Heavy Scenario</strong>: 52 marks in a 5000 lines file, call refresh 1000 times</li>
</ul>
<p>Here is the baseline that I found:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">NOTE</span><span style="color: #A6D189;"> in test/bench/bench_cache.lua</span><span style="color: #81C8BE;"> |</span><span style="color: #8CAAEE;font-style: italic;"> refresh</span><span> (x1000) - light: it</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">0010</span></span>
<span class="giallo-l"><span>mean</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">94.417ms</span><span> median</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">94.636ms</span><span> min</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">91.794ms</span><span> max</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">95.743ms</span><span> stddev</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">1.089ms</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">NOTE</span><span style="color: #A6D189;"> in test/bench/bench_cache.lua</span><span style="color: #81C8BE;"> |</span><span style="color: #8CAAEE;font-style: italic;"> refresh</span><span> (x1000) - heavy: it</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">0010</span></span>
<span class="giallo-l"><span>mean</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">95.032ms</span><span> median</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">95.293ms</span><span> min</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">92.587ms</span><span> max</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">97.148ms</span><span> stddev</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">1.431ms</span></span></code></pre>
<p>This reads as it takes around 94ms for nvim to call refresh 1000 times, on 10 attempts (or 94 micro-seconds per call). We note that light or heavy, it has little impact on the benchmark result, while a bit longer for the heavy test, it is negligible compared to the sdtdev.</p>
<p>After doing a bit of exploration I noticed that when refreshing the marks in the gutter, 2 functions where the most expensive:</p>
<ol>
<li>Function <code>vim.fn.getmarklist</code> to get the list of marks (called 3 times for local, global and special marks)</li>
<li>Function <code>vim.api.nvim_buf_set_extmark</code> used to populate the gutter (called once per mark)</li>
</ol>
<p>One common optimization pattern is to do nothing when not necessary. So I decided to remove the call to <code>nvim_buf_set_extmark</code> when unnecessary.</p>
<p>This is done by caching the last list of marks, and compare the list with the new one before refreshing the gutter. As simple as:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="lua"><span class="giallo-l"><span style="color: #CA9EE6;">function</span><span style="color: #8CAAEE;font-style: italic;"> M</span><span>.</span><span style="color: #8CAAEE;font-style: italic;">refresh</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span> …</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> local</span><span> cached_marks</span><span style="color: #81C8BE;"> =</span><span> M._marks_cache[bufnr]</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> cached_marks</span><span style="color: #81C8BE;"> and</span><span> utils.</span><span style="color: #8CAAEE;font-style: italic;">marks_equal</span><span>(marks, cached_marks) </span><span style="color: #CA9EE6;">then</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> return</span><span>;</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> end</span></span>
<span class="giallo-l"><span> </span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;"> update_extmark</span><span>(bufnr, marks)</span></span>
<span class="giallo-l"><span> </span></span>
<span class="giallo-l"><span> M._marks_cache[bufnr] </span><span style="color: #81C8BE;">=</span><span> marks</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;">end</span></span></code></pre>
<p>With this cache implementation we now have the following result:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="lua"><span class="giallo-l"><span>NOTE</span><span style="color: #CA9EE6;"> in</span><span> test</span><span style="color: #81C8BE;">/</span><span>bench</span><span style="color: #81C8BE;">/</span><span>bench_cache.lua | </span><span style="color: #8CAAEE;font-style: italic;">refresh</span><span> (x1000) with cache</span><span style="color: #81C8BE;"> -</span><span style="color: #E5C890;font-style: italic;"> light</span><span>: it</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">0010</span></span>
<span class="giallo-l"><span>mean</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">47.051</span><span>ms median</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">47.522</span><span>ms min</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">45.241</span><span>ms max</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">47.871</span><span>ms stddev</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">0.828</span><span>ms</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span>NOTE</span><span style="color: #CA9EE6;"> in</span><span> test</span><span style="color: #81C8BE;">/</span><span>bench</span><span style="color: #81C8BE;">/</span><span>bench_cache.lua | </span><span style="color: #8CAAEE;font-style: italic;">refresh</span><span> (x1000) with cache</span><span style="color: #81C8BE;"> -</span><span style="color: #E5C890;font-style: italic;"> heavy</span><span>: it</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">0010</span></span>
<span class="giallo-l"><span>mean</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">47.703</span><span>ms median</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">47.990</span><span>ms min</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">46.070</span><span>ms max</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">48.572</span><span>ms stddev</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">0.830</span><span>ms</span></span></code></pre>
<p>We immediately get a <code>2.1x</code> speed-up on this micro-benchmark.</p>
<h3 id="a-few-conclusive-notes">A few conclusive notes</h3>
<p>This only a micro-benchmark when marks are never updated, while not real it is matching the behavior where marks are rarely updated, and <code>refresh()</code> may be called many times between 2 marks update. A few more benchmark scenarios should be created with more mixed usage.</p>
<p>A few other areas to optimize I may work more in the future:</p>
<ol>
<li>When having to actually update the marks, we could simply update the required marks, instead of clearing the namespace.</li>
<li>Could we configure the aucmd to call <code>refresh()</code> in more meaningful situations? I wonder if we could update nvim to publish a <code>MarkUpdate</code> aucmd only when a marks get updated.</li>
</ol>
<hr />
<p>Now, one could ask: is it worth the effort to optimize a function that takes less than a tenth of a millisecond to execute in a text editor? I believe that, all else being equal, if all plugin authors spent time optimizing their hot loop to divide by 2 the latency with little complexity overhead, using computers would be more enjoyable day to day.</p>
Fast comment toggle in Neovim2025-09-21T00:00:00+00:002025-09-21T00:00:00+00:00
Unknown
https://dimtion.fr/blog/2025/nvim-comment-toggle/<p>I didn't see this mapping anywhere else in the community yet, so hear me out.</p>
<p>A modern <a rel="external" href="https://neovim.io/">Neovim</a> key mapping I see often these days is the use of <code><space></code> as a <code><leader></code> key. I <a rel="external" href="https://www.reddit.com/r/neovim/comments/xbigzt/why_space_as_leader/">believe</a> this is coming from the need to use a lot of indirect mappings behind the leader namespace, and provide efficient access. I don't use this mapping, I feel the default leader key <code>\</code> is close enough to the home row to stay where it is. However, in normal mode, the <code><space></code> key remains the largest key on the keyboard with little use in Vim.</p>
<p>I realized that when implementing, debugging or understanding new code, one of the most common <a rel="external" href="https://en.wikipedia.org/wiki/Read%E2%80%93eval%E2%80%93print_loop">patterns</a> I would follow is: (1) build/run code, (2) find a bug/behavior, (3) tweak the code, (4) repeat.</p>
<p>While (1) and (2) mostly depend on external tooling, optimizing (3) depends on one's text editor proficiency. One way to tweak the code efficiently is by commenting out parts of the code. This allows for quickly changing the behavior, trying an old or new expression, toggling a specific unit test case, enabling/disabling a new feature, enable/disable flags.</p>
<p>While I don't advocate keeping commented code in final commits, commenting code in and out is a super efficient way to draft code changes. Additionally, commenting in and out code keeps all the short-term history of the code right there in the buffer. So here is my simple remap I've been using for a while now:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="lua"><span class="giallo-l"><span>vim.keymap.</span><span style="color: #8CAAEE;font-style: italic;">set</span><span>({</span><span style="color: #A6D189;">"n"</span><span>}, </span><span style="color: #A6D189;">"<space>"</span><span>, </span><span style="color: #A6D189;">"gcc"</span><span>, { silent</span><span style="color: #81C8BE;"> =</span><span style="color: #E78284;"> true</span><span> })</span></span>
<span class="giallo-l"><span>vim.keymap.</span><span style="color: #8CAAEE;font-style: italic;">set</span><span>({</span><span style="color: #A6D189;">"v"</span><span>}, </span><span style="color: #A6D189;">"<space>"</span><span>, </span><span style="color: #A6D189;">"gc"</span><span>, { silent</span><span style="color: #81C8BE;"> =</span><span style="color: #E78284;"> true</span><span> })</span></span></code></pre>
<p>Space bar as comment-in comment-out is such a good muscle memory, I do believe I'm directly more efficient in editing code day to day. Given how rarely I see this mapping, I would advocate for more people trying this mapping in the community.</p>
<p>Combine it with <code>gv</code> (redo previous visual selection) and <a rel="external" href="https://github.com/nvim-treesitter/nvim-treesitter-textobjects/tree/main">Treesitter text objects</a> <code>ac</code>, <code>ic</code> to comment lines at the speed of light:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="lua"><span class="giallo-l"><span>vim.keymap.</span><span style="color: #8CAAEE;font-style: italic;">set</span><span>({ </span><span style="color: #A6D189;">"x"</span><span>, </span><span style="color: #A6D189;">"o" </span><span>}, </span><span style="color: #A6D189;">"ac"</span><span>, </span><span style="color: #CA9EE6;">function</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;"> require</span><span style="color: #A6D189;"> "nvim-treesitter-textobjects.select"</span><span>.</span><span style="color: #8CAAEE;font-style: italic;">select_textobject</span><span>(</span><span style="color: #A6D189;">"@comment.outer"</span><span>, </span><span style="color: #A6D189;">"textobjects"</span><span>)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;">end</span><span>)</span></span>
<span class="giallo-l"><span>vim.keymap.</span><span style="color: #8CAAEE;font-style: italic;">set</span><span>({ </span><span style="color: #A6D189;">"x"</span><span>, </span><span style="color: #A6D189;">"o" </span><span>}, </span><span style="color: #A6D189;">"ic"</span><span>, </span><span style="color: #CA9EE6;">function</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;"> require</span><span style="color: #A6D189;"> "nvim-treesitter-textobjects.select"</span><span>.</span><span style="color: #8CAAEE;font-style: italic;">select_textobject</span><span>(</span><span style="color: #A6D189;">"@comment.outer"</span><span>, </span><span style="color: #A6D189;">"textobjects"</span><span>)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;">end</span><span>)</span></span></code></pre>
<hr />
<p>Few examples of work in progress code changes, or <em>things you would not see committed</em>. That I see myself very often do thanks how easy <code><space></code> is accessible.</p>
<p>Trying the <code>setup()</code> function instead of the <code>init()</code> function:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="lua"><span class="giallo-l"><span style="color: #CA9EE6;">function</span><span style="color: #8CAAEE;font-style: italic;"> configure</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> -- require("guttermarks").init()</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;"> require</span><span>(</span><span style="color: #A6D189;">"guttermarks"</span><span>).</span><span style="color: #8CAAEE;font-style: italic;">setup</span><span>()</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;">end</span></span></code></pre>
<p>Temporarily disable unit test case to restrict debugging to one function:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="lua"><span class="giallo-l"><span style="color: #949CBB;font-style: italic;">-- T["alphabet_lower"]["is_lower"] = function(x)</span></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;">-- eq(utils.is_lower(x), true)</span></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;">-- end</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span>T[</span><span style="color: #A6D189;">"alphabet_lower"</span><span>][</span><span style="color: #A6D189;">"is_upper"</span><span>] </span><span style="color: #81C8BE;">=</span><span style="color: #CA9EE6;"> function</span><span style="color: #949CBB;">(</span><span style="color: #EA999C;font-style: italic;">x</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;"> eq</span><span>(utils.</span><span style="color: #8CAAEE;font-style: italic;">is_upper</span><span>(x), </span><span style="color: #E78284;">false</span><span>)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;">end</span></span></code></pre>
<p>Give it a try for a week, tell me what you think!</p>
GutterMarks.nvim2025-08-03T00:00:00+00:002025-08-03T00:00:00+00:00
Unknown
https://dimtion.fr/blog/2025/guttermarks-nvim/<h1 id="guttermarks-nvim">GutterMarks.nvim</h1>
<p>I love <a rel="external" href="https://neovim.io/doc/user/motion.html#_7.-marks">Neovim marks</a>, this weekend, I got tired by existing neovim marks plugins. They are often complex, impose new workflows and sometimes don't provide the basics very well out of the box: display the Marks in the buffer gutter.</p>
<p>For example, <a rel="external" href="https://github.com/chentoast/marks.nvim">marks.nvim</a> runs a scheduled function every 150 milliseconds to refresh the existing marks, both not great for performances and slow. Further, I don't need all the other fancy features.</p>
<p>So I wrote <a rel="external" href="https://github.com/dimtion/guttermarks.nvim">GutterMarks.nvim</a>.</p>
<p>The pitch is simple: display in the Neovim gutter the existing Marks for a given buffer.</p>
<p><img src="https://dimtion.fr/blog/2025/guttermarks-nvim/screenshot_01.png" alt="GutterMarks screenshot" /></p>
<p>There are a few <a rel="external" href="https://github.com/dimtion/guttermarks.nvim/blob/main/lua/guttermarks/default_config.lua">configuration options</a>:</p>
<ul>
<li>Display Local, Global and Special marks (<code>'."^<>[]0-9</code>)</li>
<li>Change the highlight groups (and thus display colors)</li>
<li>Exclude filetypes and buffer types</li>
</ul>
<p>And that's it. Nothing fancy.</p>
<p>Installation is simple using <a rel="external" href="https://github.com/folke/lazy.nvim">lazy.nvim</a></p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="lua"><span class="giallo-l"><span>{</span></span>
<span class="giallo-l"><span style="color: #A6D189;"> "dimtion/guttermarks.nvim"</span><span>,</span></span>
<span class="giallo-l"><span> event</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "VeryLazy"</span><span>,</span></span>
<span class="giallo-l"><span>}</span></span></code></pre>
<p>Using <a rel="external" href="https://github.com/rose-pine/neovim">Rosé Pine</a> theme, I'm changing the gutter marks color using the <a rel="external" href="https://github.com/dimtion/dotfiles/blob/master/.config/nvim/lua/dimtion/plugins/theme.lua">highlight groups override</a>:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="lua"><span class="giallo-l"><span> {</span></span>
<span class="giallo-l"><span style="color: #A6D189;"> "rose-pine/neovim"</span><span>,</span></span>
<span class="giallo-l"><span> name</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "rose-pine"</span><span>,</span></span>
<span class="giallo-l"><span> lazy</span><span style="color: #81C8BE;"> =</span><span style="color: #E78284;"> false</span><span>,</span></span>
<span class="giallo-l"><span> priority</span><span style="color: #81C8BE;"> =</span><span style="color: #EF9F76;"> 1000</span><span>,</span></span>
<span class="giallo-l"><span> opts</span><span style="color: #81C8BE;"> =</span><span> {</span></span>
<span class="giallo-l"><span> highlight_groups</span><span style="color: #81C8BE;"> =</span><span> {</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> -- Blend colours against the "base" background</span></span>
<span class="giallo-l"><span> CursorLine</span><span style="color: #81C8BE;"> =</span><span> { bg</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "pine"</span><span>, blend</span><span style="color: #81C8BE;"> =</span><span style="color: #EF9F76;"> 30</span><span> },</span></span>
<span class="giallo-l"><span> </span></span>
<span class="giallo-l"><span style="color: #E78284;"> ...</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span> GutterMarksLocal</span><span style="color: #81C8BE;"> =</span><span> { fg</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "muted" </span><span>},</span></span>
<span class="giallo-l"><span> GutterMarksGlobal</span><span style="color: #81C8BE;"> =</span><span> { fg</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "muted" </span><span>},</span></span>
<span class="giallo-l"><span> GutterMarksSpecial</span><span style="color: #81C8BE;"> =</span><span> { fg</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "muted" </span><span>},</span></span>
<span class="giallo-l"><span> },</span></span>
<span class="giallo-l"><span> },</span></span>
<span class="giallo-l"><span> },</span></span>
<span class="giallo-l"></span></code></pre>
<hr />
<p>I'm not planning to add much other features, in particular to keep the scope small. But if you find any bugs or nice to have improvements, don't hesitate to file a <a rel="external" href="https://github.com/dimtion/guttermarks.nvim/issues">bug report</a>. Hope this plugin will find other happy users!</p>
LLM agents as a new automation level2025-07-13T00:00:00+00:002025-07-13T00:00:00+00:00
Unknown
https://dimtion.fr/blog/2025/llm-agent-oncall/<p>In high load teams, there is this common thread that appears again and again: "we don't have time to fully automate this SOP". This happens for problems that are rare (a few time a year), are low impact (only 0.0x% of clients/requests/$ impacted) or high engineering cost to fix. Usually, if your organization is effective, you'll take note of the recurring problem, add it to a backlog and maybe provide a proper "long term fix" for the next sprint. Oftentimes this problem stays in the backlog forever.</p>
<p>In very productive teams, senior engineers and management fosters a culture empowering the engineers to semi-automate the mitigation for this type of recurring problems, it could be a quick python script in a shared folder, a well written SOP for newcomers, or simply tune the alert level if there is no action item.</p>
<p>This is beautifully summarized in the now <a rel="external" href="https://xkcd.com/1205/">famous XKCD</a>:</p>
<p><img src="https://dimtion.fr/blog/2025/llm-agent-oncall/is_it_worth_the_time_2x.png" alt="" /></p>
<center><i>Nobody cares about the top right quadrant</i></center>
<p>I'd like to make the case with LLM-agent tools coming into the engineering toolkit, effective teams could leverage, them to tackle issues that were too expensive to tackle today, which could effectively improve the quality of tech products for the users.</p>
<hr />
<p>Applying the XKCD table to engineering oncall issue mitigation, we can rank the ways of solving a recurring issue in the following order:</p>
<p><img src="https://dimtion.fr/blog/2025/llm-agent-oncall/llm-agent-oncall.svg" alt="" /></p>
<p>On the left, manual mitigations are cheap do to once, this is when the oncall will bounce the fleet, or manually update a configuration to partially mitigate the issue, this can be done in a short amount of time (~15 minutes). Sometimes, this action is also a loosely defined set of investigation steps, that require fetching, combining data from multiple source before taking a decision. It generally only fixes the issue once, and the same action might be needed again if the issue happens again.</p>
<p>In between, there are semi-automated, or "scripted mitigation". Scripted mitigations usually are created by team when a recurring issue pass a certain threshold of either complexity (too many items to fix) or a threshold of frequency. Scripts take more time to write the first time, from a few minutes to a few hours, but once created, they can save substantial time on subsequent occurrences. As examples: scripts to re-drive a few asynchronous events, scripts to update multiple objects in a database in an invalid state.</p>
<p>Finally, generally, the preferred approach is a systematic bug fix or system redesign to make the error case fully disappear. Such solution range take from hours, days, or weeks of work to be implemented, depending on the complexity. For rare, low-impact issues (that still impact a few unhappy users), this type of fix is rare prioritized, unless another higher impact issue allow the team to bundle the bug fix with the other issue.</p>
<p>In a perfect world, teams would reach for this last solution as much as possible, but time-constrains and other priorities often time only allow to reach for a proper resolution only if the impact pass a certain threshold of impact.</p>
<hr />
<p>The thesis I'm bringing is that LLM based tools which are getting better and better at writing code snippets, reading and compiling from unstructured data source can be (and should be) used as a new automation level in operational excellence. On the OPEX/CAPEX spectrum, such tools allow us to close the gap between manual incident mitigation and automated mitigation:</p>
<p><img src="https://dimtion.fr/blog/2025/llm-agent-oncall/llm-agent-oncall-genai.svg" alt="" /></p>
<p>LLM agents can be seen as a new tool for operational teams, serving as helpers for mitigating low-impact issues while software engineers focus most of their attention on higher-impact work during busy on-call rotations.</p>
<p>When writing a mitigation script would have taken a couple of hours in the past (with proper testing, edge case handling, etc.), LLM agents can give engineers a head start. They write the boilerplate, handle the core business logic of the script, and generate data for script testing before oncall can validate and execute it in a production environment. For an issue that impacts only a few user a month, the oncall Engineer may never had the time to investigate the issue. Or if they could investigate the issue, they may have judged that performing a mitigation or fix would be too much effort compared to the impact, thus leaving the issue unmitigated or partially mitigated.</p>
<p>Another task that takes time for software engineers is the process of following loosely defined SOPs. These SOPs are can sometime be vague, and require investigation work (otherwise they would have been automated sic). With proper guardrails, some investigation SOPs (checking multiple log streams, databases, API endpoints…) can now be half-automated. The LLM agent to do most of the legwork of combining unstructured data sources, summarizing issues, bucketizing problems, and report the findings to the oncall. In this case, the LLM serves as an approximate interpreter of the SOP, something that was qualitatively impossible in the past.</p>
<p>Note that high-impact operational work still benefit from GenAI tools (and for similar reasons). The point I'm making here is, by reducing the cost of investigating, analyzing and mitigating low-impact issues, LLM may be a tool that ultimately improve the overall software operational quality for high TPS systems. Such systems, often accumulate over time a big backlog of detected, but unmitigated bugs, which if were fully tackled would generally improve the service quality in a measurable way.</p>
<hr />
<p>Finally, please consider that I'm not advocating in leaving LLM agents loose access to production environment. Before adding any agent to their workflow, teams should assess their security posture, especially in the <a rel="external" href="https://simonwillison.net/2025/Jun/16/the-lethal-trifecta/">new class of security issues</a> that LLM agents bring security engineering. Further, as an industry, we should recognize that LLM tools are <a rel="external" href="https://metr.org/blog/2025-07-10-early-2025-ai-experienced-os-dev-study/">not always a net productivity gain</a>, and the same should apply when designing an agent optimized for on-call. Hopefully good engineering practices will emerge with time.</p>
Zellij2025-05-10T00:00:00+00:002025-05-10T00:00:00+00:00
Unknown
https://dimtion.fr/wiki/linux/zellij/<p>Cache location in macos:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"><span style="color: #81C8BE;">~</span><span>/Library/Caches/org.Zellij-Contributors.Zellij</span></span></code></pre><h2 id="bug-with-session-dissapearing-after-an-update">Bug with session dissapearing after an update</h2>
<p>See: <a rel="external" href="https://github.com/zellij-org/zellij/issues/3371">zellij#3371</a></p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># kill all running process</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">ps</span><span style="color: #A6D189;"> aux</span><span style="color: #81C8BE;"> |</span><span style="color: #8CAAEE;font-style: italic;"> grep</span><span style="color: #A6D189;"> -i "[z]ellij --server"</span><span style="color: #81C8BE;"> |</span><span style="color: #8CAAEE;font-style: italic;"> awk</span><span style="color: #A6D189;"> {'print $2'}</span><span style="color: #81C8BE;"> |</span><span style="color: #8CAAEE;font-style: italic;"> xargs</span><span style="color: #A6D189;"> kill</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># Copy the old version to the new version cache:</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">cp</span><span style="color: #A6D189;"> -r ~/Library/Caches/org.Zellij-Contributors.Zellij/{0.41.2/,0.42.1}</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># List new sessions</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">zellij</span><span style="color: #A6D189;"> ls</span></span></code></pre>Pytorch2025-04-06T00:00:00+00:002025-04-06T00:00:00+00:00
Unknown
https://dimtion.fr/wiki/ml/pytorch/<h2 id="pytorch-for-m-series-mac-metal-accelerated">Pytorch for M series mac (Metal accelerated)</h2>
<h3 id="using-bare-pip">Using bare pip</h3>
<p>From <a rel="external" href="https://developer.apple.com/metal/pytorch/">Apple documentation</a>, using simple pip:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">pip3</span><span style="color: #A6D189;"> install --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cpu</span></span></code></pre><h3 id="for-a-uv-project">For a uv project</h3>
<p>For a Python project using <a rel="external" href="https://github.com/astral-sh/uv">uv</a> as a project and package manager do the following:</p>
<p>Add to the uv <code>pyproject.toml</code> file:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="toml"><span class="giallo-l"><span style="color: #949CBB;">[</span><span>tool.uv.sources</span><span style="color: #949CBB;">]</span></span>
<span class="giallo-l"><span>torch</span><span style="color: #81C8BE;"> =</span><span style="color: #949CBB;"> {</span><span> index</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "pytorch"</span><span style="color: #949CBB;"> }</span></span>
<span class="giallo-l"><span>torchvision</span><span style="color: #81C8BE;"> =</span><span style="color: #949CBB;"> {</span><span> index</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "pytorch"</span><span style="color: #949CBB;"> }</span></span>
<span class="giallo-l"><span>torchaudio</span><span style="color: #81C8BE;"> =</span><span style="color: #949CBB;"> {</span><span> index</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "pytorch"</span><span style="color: #949CBB;"> }</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;">[[</span><span>tool.uv.index</span><span style="color: #949CBB;">]]</span></span>
<span class="giallo-l"><span>name</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "pytorch"</span></span>
<span class="giallo-l"><span>url</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "https://download.pytorch.org/whl/nightly/cpu"</span></span>
<span class="giallo-l"><span>explicit</span><span style="color: #81C8BE;"> =</span><span style="color: #EF9F76;"> true</span></span></code></pre>
<p>Then run:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="plain"><span class="giallo-l"><span>uv add --prerelease allow torch torchvision torchaudio</span></span></code></pre>
<p>There is an <a rel="external" href="https://github.com/astral-sh/uv/issues/8341">open issue</a> on uv github to improve a bit the CLI to automatically add explicit indexes to the <code>pyproject.toml</code> file.</p>
<h3 id="for-a-simple-script-one-time-projec-using-uv">For a simple script/one time projec using uv</h3>
<p>Run the command:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">uv</span><span style="color: #A6D189;"> add --prerelease allow torch torchvision torchaudio --index pytorch=https://download.pytorch.org/whl/nightly/cpu</span></span></code></pre>ComfyUI RAW Image Loader2025-03-30T00:00:00+00:002025-03-30T00:00:00+00:00
Unknown
https://dimtion.fr/blog/2025/comfyui-raw-image/<p>This weekend I was trying a bit <a rel="external" href="https://www.comfy.org/">ComfyUI</a> to see what is happening on the stable diffusion domain. Even though it looks like <a rel="external" href="https://openai.com/index/introducing-4o-image-generation/">stable diffusion is no longer the best and greatest</a>, it is still clearly an impressive tool to generate images. Most users seem to focus on using it to either create<a rel="external" href="https://docs.comfy.org/tutorials/basic/text-to-image"> image from text</a>, do <a rel="external" href="https://docs.comfy.org/tutorials/basic/image-to-image">style transfer</a>, or plenty of uses cases that are trendy on Twitter and other social media.</p>
<p>One thing I wanted to try if it is possible to use the workflows to do photo editing, and in particular, RAW image editing. It turns out that there is no plugin that exist to directly load a RAW image and insert it into a ComfyUI workflow. Most of the internet mostly care about jpeg and png images, I'm guessing.</p>
<p>More generally it seems that most image generation models are not meant for RAW image editing, which would be a nice project to explore another day. Anyways, as a first step I decided to experiment with ComfyUI API and came up with the most minimal possible extension: <a rel="external" href="https://github.com/dimtion/comfyui-raw-image">ComfyUI Raw Image Loader</a>.</p>
<p>This extension provides a simple node to load a RAW image from disk and convert it to a proper Torch tensor (or simply an <code>IMAGE</code> in ComfyUI vocabulary) that can then be used as input for any image-to-X workflow. I added a couple of features to allow brightness adjustment and how it should behave with brightness clipping. I was able to set it up quickly thanks to the amazing <a rel="external" href="https://letmaik.github.io/rawpy/api/rawpy.RawPy.html">pyraw</a> library.</p>
<p><img src="https://dimtion.fr/blog/2025/comfyui-raw-image/2025-03-30_comfyui-node-screenshot.png" alt="ComfyUI Load Raw Image node" /></p>
<p>The overall code is very simple, ComfyUI has a good documentation to create a new node, in short, the code can be summarized to:</p>
<ol>
<li>Create a Python class</li>
<li>Define the return and input types, with their constrains and validations</li>
<li>Write a main function that takes those inputs and return the specified output</li>
</ol>
<p>Here is a skeleton example of how to create a node extension:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="python"><span class="giallo-l"><span style="color: #CA9EE6;">class</span><span style="color: #E5C890;font-style: italic;"> LoadRawImage</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> # Define node interface metadata</span></span>
<span class="giallo-l"><span> RETURN_TYPES</span><span style="color: #81C8BE;"> =</span><span style="color: #949CBB;"> (</span><span style="color: #A6D189;">"IMAGE"</span><span style="color: #949CBB;">,)</span><span style="color: #949CBB;font-style: italic;"> # What the node outputs</span></span>
<span class="giallo-l"><span> FUNCTION</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "load_img"</span><span style="color: #949CBB;font-style: italic;"> # Main function to call</span></span>
<span class="giallo-l"><span> CATEGORY</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "image"</span><span style="color: #949CBB;font-style: italic;"> # UI category</span></span>
<span class="giallo-l"><span> DESCRIPTION</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "Load a RAW image into ComfyUI."</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #EF9F76;font-style: italic;"> @</span><span style="color: #CA9EE6;font-style: italic;">classmethod</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> def</span><span style="color: #8CAAEE;font-style: italic;"> INPUT_TYPES</span><span style="color: #949CBB;">(</span><span style="color: #EA999C;font-style: italic;">cls</span><span style="color: #949CBB;">):</span></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> # Define UI inputs (required and optional)</span></span>
<span class="giallo-l"><span> files</span><span style="color: #81C8BE;"> =</span><span style="color: #8CAAEE;"> get_files_from_input_directory</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> return</span><span style="color: #949CBB;"> {</span></span>
<span class="giallo-l"><span style="color: #A6D189;"> "required"</span><span style="color: #949CBB;">: {</span><span style="color: #A6D189;">"image"</span><span style="color: #949CBB;">: (</span><span>files</span><span style="color: #949CBB;">, {</span><span style="color: #A6D189;">"image_upload"</span><span style="color: #949CBB;">:</span><span style="color: #EF9F76;"> True</span><span style="color: #949CBB;">})},</span></span>
<span class="giallo-l"><span style="color: #A6D189;"> "optional"</span><span style="color: #949CBB;">: {</span></span>
<span class="giallo-l"><span style="color: #A6D189;"> "use_auto_bright"</span><span style="color: #949CBB;">: (</span><span style="color: #A6D189;">"BOOLEAN"</span><span style="color: #949CBB;">, {</span><span style="color: #A6D189;">"default"</span><span style="color: #949CBB;">:</span><span style="color: #EF9F76;"> True</span><span style="color: #949CBB;">}),</span></span>
<span class="giallo-l"><span style="color: #A6D189;"> "bright_adjustment"</span><span style="color: #949CBB;">: (</span><span style="color: #A6D189;">"FLOAT"</span><span style="color: #949CBB;">, {</span><span style="color: #A6D189;">"default"</span><span style="color: #949CBB;">:</span><span style="color: #EF9F76;"> 1.0</span><span style="color: #949CBB;">,</span><span style="color: #A6D189;"> "min"</span><span style="color: #949CBB;">:</span><span style="color: #EF9F76;"> 0.1</span><span style="color: #949CBB;">,</span><span style="color: #A6D189;"> "max"</span><span style="color: #949CBB;">:</span><span style="color: #EF9F76;"> 3.0</span><span style="color: #949CBB;">}),</span></span>
<span class="giallo-l"><span style="color: #A6D189;"> "highlight_mode"</span><span style="color: #949CBB;">: ([</span><span style="color: #A6D189;">"clip"</span><span style="color: #949CBB;">,</span><span style="color: #A6D189;"> "blend"</span><span style="color: #949CBB;">], {</span><span style="color: #A6D189;">"default"</span><span style="color: #949CBB;">:</span><span style="color: #A6D189;"> "clip"</span><span style="color: #949CBB;">})</span></span>
<span class="giallo-l"><span style="color: #949CBB;"> }</span></span>
<span class="giallo-l"><span style="color: #949CBB;"> }</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #EF9F76;font-style: italic;"> @</span><span style="color: #CA9EE6;font-style: italic;">classmethod</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> def</span><span style="color: #8CAAEE;font-style: italic;"> VALIDATE_INPUTS</span><span style="color: #949CBB;">(</span><span style="color: #EA999C;font-style: italic;">cls</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> image</span><span style="color: #949CBB;">):</span></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> # Validate inputs before processing</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> return</span><span style="color: #EF9F76;"> True</span><span style="color: #CA9EE6;"> or</span><span> error_message</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #EF9F76;font-style: italic;"> @</span><span style="color: #CA9EE6;font-style: italic;">classmethod</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> def</span><span style="color: #8CAAEE;font-style: italic;"> IS_CHANGED</span><span style="color: #949CBB;">(</span><span style="color: #EA999C;font-style: italic;">cls</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> image</span><span style="color: #949CBB;">):</span></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> # Track if inputs have changed (for caching)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> return</span><span> hash_of_input</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> def</span><span style="color: #8CAAEE;font-style: italic;"> load_img</span><span style="color: #949CBB;">(</span><span style="color: #E78284;font-style: italic;">self</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> image</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> use_auto_bright</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">True</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> bright_adjustment</span><span style="color: #81C8BE;">=</span><span style="color: #EF9F76;">1.0</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> highlight_mode</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">"clip"</span><span style="color: #949CBB;">):</span></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> # Main processing function that matches FUNCTION name</span></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> # Process inputs and return outputs matching RETURN_TYPES</span></span>
<span class="giallo-l"><span> processed_image</span><span style="color: #81C8BE;"> =</span><span style="color: #8CAAEE;"> process_raw_image</span><span style="color: #949CBB;">(</span><span>image</span><span style="color: #949CBB;">,</span><span> parameters...</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> return</span><span style="color: #949CBB;"> (</span><span>processed_image</span><span style="color: #949CBB;">,)</span><span style="color: #949CBB;font-style: italic;"> # Return as tuple matching RETURN_TYPES</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># Register node(s) for ComfyUI to discover</span></span>
<span class="giallo-l"><span>NODE_CLASS_MAPPINGS</span><span style="color: #81C8BE;"> =</span><span style="color: #949CBB;"> {</span><span style="color: #A6D189;">"Load Raw Image"</span><span style="color: #949CBB;">:</span><span> LoadRawImage</span><span style="color: #949CBB;">}</span></span></code></pre>
<p>I wonder if ComfyUI explored the option to use Python type annotations instead of simple strings to define inputs and outputs. At least the string approach has the advantage of being simple and universal.</p>
<p>In the end, I created a simple <a rel="external" href="https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0">SDXL</a> workflow, using a refiner to change a little bit the style of an input picture. I'm experimenting a bit more to see how to generate more complex changes, and provide more control over the input image.</p>
<p><img src="https://dimtion.fr/blog/2025/comfyui-raw-image/2025-03-30_comfyui-workflow-screenshot.png" alt="Example ComfyUI workflow" /></p>
<p>I do believe diffusion, LLMs and more generally deep learning tools, are good tools overall for people to be able to experiment too, and in a time where artists and photographers feel threaten by them, providing more tools for said people to leverage GenAI tools, will definitively be good to propagate knowledge.</p>
<p>See: <a rel="external" href="https://github.com/dimtion/comfyui-raw-image">ComfyUI Raw Image Loader</a></p>
Metonymies in software engineering2025-03-09T00:00:00+00:002025-03-09T00:00:00+00:00
Unknown
https://dimtion.fr/blog/2025/metonymy-swe/<p>Properly communicating in engineering is often a tradeoff between being concise and being accurate. A common tool to remain concise while remaining specific is to use <a rel="external" href="https://en.wikipedia.org/wiki/Metonymy">metonymies</a>. A metonymy is a figure of speech in which a specific concept is referred by another very close word.</p>
<p>A common type of metonymy used is to designate a concept by a specific implementation it. For example, when one means "We should use Redis to solve problem A" do they mean:</p>
<ul>
<li>We need an key value store?</li>
<li>We need an in-memory database?</li>
<li>We need a distributed database?</li>
<li>We need a (distributed) cache?</li>
<li>We need a database with feature x (sets, hyperloglog…)</li>
</ul>
<p>Redis is arguably all of those concepts at the same time. If you are sure your co-worker (or blog reader) understands which capacity of this technology you refer to, go ahead, use the word <em>Redis</em>. However, be sure to insist which aspect of <em>Redis</em> solves problem A.</p>
<p>This is mostly a problem in the following situations:</p>
<ul>
<li>Some junior engineers use metonymies by lack of exposure of multiple implementations of a concept.</li>
<li>Confusion within the team of what the engineer exactly mean (do they refer to a key/value store or a in-memory database?)</li>
<li>Using a specific technology name to often can corner one's mental model into this specific technology instead of the more general concept.</li>
</ul>
<p>Some other examples of common metonymies that might cause confusion:</p>
<ul>
<li>Hashmap for a Map</li>
<li>DynamoDB for a document database</li>
<li>PostgreSQL for SQL database</li>
<li>ReactJS for Javascript</li>
</ul>
<ul>
<li>Github Actions for CI/CD</li>
</ul>
<p>Overall, when the concept is properly understood there is little harm to refer to it by using a specific implementation, but be aware of the mental model your listeners might have of this concept, and more importantly, question your assumptions to makes sure you don't corner your thoughts into a specific implementation.</p>
Falsehoods Software Developers Believe About Event-Driven Systems2024-06-30T00:00:00+00:002024-06-30T00:00:00+00:00
Unknown
https://dimtion.fr/blog/falsehoods-event-driven/<p>When building a distributed system, a common design pattern is to follow the
event-driven approach. Event-driven systems can range from a simple in-memory
queue to a serverless AWS Lambda with a preceding queue, or even connected
Kafka clusters. when reviewing code implementing an even driven architecture, I
see common mistakes that cause toil or even operational incidents once deployed
to production.</p>
<p>Here are unordered misconceptions developers have about event-driven
architectures. Use this as a checklist for design and code review.</p>
<h2 id="message-ordering">Message ordering</h2>
<ul>
<li>Events will arrive in order</li>
<li>Events will arrive in order, even with a single consumer</li>
<li>Events will arrive in order, even if specified by the producer contract</li>
<li>Events will arrive in order, even with days between messages</li>
<li>Events can always be ordered</li>
</ul>
<h2 id="message-duplication">Message duplication</h2>
<ul>
<li>Events won't be duplicated</li>
<li>Events won't be duplicated, in at-most-once delivery queues</li>
<li>Events won't be duplicated, even if specified by producer contract</li>
<li>Events won't be duplicated, even with de-duplication upon arrival</li>
</ul>
<h2 id="idempotency">Idempotency</h2>
<ul>
<li>Adding an idempotency key ensures idempotency</li>
<li>Equal idempotency keys mean identical payloads</li>
<li>Event timestamp is a valid idempotency key</li>
<li>Writing idempotent code is easy</li>
<li>Maintain and improve idempotent code is easy</li>
<li>Idempotency can be solved via adding a distributed lock and an idempotency
key</li>
</ul>
<h2 id="load-management">Load management</h2>
<ul>
<li>Low TPS systems are not subject to backlog</li>
<li>Low TPS systems are not subject throttling</li>
<li>Processing timeout of XX seconds is sufficient</li>
<li>Processing timeout of XX minutes is sufficient</li>
<li>Processing timeout of XX hours is sufficient</li>
<li>Upstream dependencies' timeouts are properly configured</li>
<li>Retry policy is properly configured</li>
<li>Event processing time is constant and performance will remain consistent under load</li>
</ul>
<h2 id="producer-contract">Producer contract</h2>
<ul>
<li>Event producer can be trusted to always produce valid events</li>
<li>Event producer can be trusted to generally produce valid events</li>
<li>Event producer can be trusted to produce non-conflicting events</li>
<li>Event producer can be trusted to not overload the consumer</li>
<li>Event producer can be trusted to send message on time</li>
<li>Event producer can be trusted to never fail</li>
<li>Event producer can be trusted to rarely fail</li>
<li>Event producer can be trusted, even if it is an internal process</li>
</ul>
<h2 id="consumer-contract">Consumer contract</h2>
<ul>
<li>Event consumer is simple enough to never fail</li>
<li>Event consumer does not need a scaling strategy</li>
<li>Event consumer downstream dependencies support idempotent calls</li>
<li>Events will never be dropped</li>
</ul>
<h2 id="dead-letter-queues">Dead letter queues</h2>
<ul>
<li>Dead letter queues are not necessary</li>
<li>Dead letter queue is properly configured</li>
<li>Only a handful events will end up in the dead letter queue</li>
<li>Even if there are many events in the DLQ, there are only a few
representative error categories</li>
<li>At least it will be easy to sort out the different error categories</li>
</ul>
<h2 id="recovery">Recovery</h2>
<ul>
<li>System does not need manual recovery</li>
<li>Manual recovery won't require modifying events</li>
<li>Manual recovery won't require the producer to regenerate messages</li>
<li>Manual recovery won't coincide with another system failure</li>
<li>Manual recovery will be completed within minutes</li>
<li>Manual recovery will be completed within hours</li>
<li>Manual recovery will be completed within days</li>
<li>Pending manual recovery, events can simply stay in the queue</li>
</ul>
<h2 id="architecture">Architecture</h2>
<ul>
<li>Orchestrated architectures are better and simpler</li>
<li>Choreographed architectures are better and simpler</li>
<li>Unbounded queues are better than bounded queues</li>
<li>Event-driven architecture are simpler to reason about</li>
</ul>
<p><em>[list to be updated]</em></p>
Trying out lazy.nvim2024-06-05T00:00:00+00:002024-06-05T00:00:00+00:00
Unknown
https://dimtion.fr/blog/trying-out-lazy-nvim/<p>During the few months, following the news that
<a rel="external" href="https://github.com/wbthomason/packer.nvim/commit/16c5798b4968669174c76e978944a2c82653fe67">packer.nvim</a>
is deprecated, I wanted to try out the new cool package manager in town
<a rel="external" href="https://github.com/folke/lazy.nvim">lazy.nvim</a>. I spent a bit of time
migrating over my
<a rel="external" href="https://github.com/dimtion/dotfiles/tree/master/.config/nvim">personal</a> and
work configurations over to lazy. Here are some thoughts I had during the
migration.</p>
<p>To give a bit of context, my journey on <code>/n?vim/</code> package managers is the
following:</p>
<ol>
<li>Vim no package manager</li>
<li>Vim + vim-plug</li>
<li>Neovim + vim-plug (VimL)</li>
<li>Neovim + packer.nvim (Lua)</li>
<li>Neovim + Lazy.nvim (Lua)</li>
</ol>
<h2 id="the-good">The good</h2>
<p>On the good part and as a first impression, lazy.nvim setup and installation is
pretty simple, just copy the lines of the self-bootstrap code, restart nvim and
it downloads and install lazy flawlessly. In case you have any issue
<code>:checkhealth lazy</code> has good self-check that provides insightful debugging
information. This is definitively on par with packer.nvim and from memory even
working better than vim-plug.</p>
<p>Perhaps the best tool lazy.nvim provides is its UI. It is very practical,
information dense and is very easy to navigate. Launch <code>:Lazy</code> and just follow
the UI shortcuts to jump around between which lazy-loaded plugins are launched.
Which ones are not loaded yet and what shortcut or event will launch them. The
profiler is a great tool to understand which plugin has a strong impact on
startup-time, and decide which plugins are worth spending time configure to
lazy load.</p>
<p><img src="https://dimtion.fr/blog/trying-out-lazy-nvim/lazy-cmd-screenshot.png" alt="Running :lazy" />
Also, the pining of dependencies version is a game-changer, copying from modern
package managers, it helps a lot with sharing a configuration across multiple
machines. Pining plus headless nvim, allows updating the packages in <a rel="external" href="https://github.com/dimtion/dotfiles/blob/master/Makefile">one
command line</a>,
even in a CI environment, and make sure the same version of all packages are running everywhere.</p>
<p>To update the neovim configuration, one can run <code>:Lazy</code> then press <code>S</code> to "sync" the latest dependencies, pretty convenient. alternatively, my preferred approach is to use Makefile and have this formula:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="make"><span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">update</span><span style="color: #81C8BE;">:</span></span>
<span class="giallo-l"><span> XDG_CONFIG_HOME=./.config nvim --headless "+Lazy! update" +qa</span></span>
<span class="giallo-l"><span> XDG_CONFIG_HOME=./.config nvim --headless "+TSUpdateSync" +qa</span></span></code></pre><h2 id="the-can-improve">The 'can improve'</h2>
<p>Lazy.nvim taking a radically different approach on configuration of plugins
than other more common plugin managers, it requires a mental shift in how to
configure each plugin. Other package managers (such as plug or packer)
typically have one location where the plugins are declared, and another one
where they are configured, those package managers leverage vim plugin
infrastructure which is quite standard and common nowadays.</p>
<p>It results in configurations that look like such:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="lua"><span class="giallo-l"><span style="color: #949CBB;font-style: italic;">-- 1. Declare loaded plugins</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">require</span><span>(</span><span style="color: #A6D189;">'packer'</span><span>).</span><span style="color: #8CAAEE;font-style: italic;">startup</span><span>(</span><span style="color: #CA9EE6;">function</span><span style="color: #949CBB;">(</span><span style="color: #EA999C;font-style: italic;">use</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;"> use</span><span style="color: #A6D189;"> "mbbill/undotree"</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;">end</span><span>)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;">-- 2. Configure each plugin using plugin README:</span></span>
<span class="giallo-l"><span>vim.keymap.</span><span style="color: #8CAAEE;font-style: italic;">set</span><span>(</span><span style="color: #A6D189;">"n"</span><span>, </span><span style="color: #A6D189;">"<leader><u>"</span><span>, vim.cmd.UndotreeToggle, {</span></span>
<span class="giallo-l"><span> desc</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "Show nvim undotree"</span><span>,</span></span>
<span class="giallo-l"><span>})</span></span></code></pre>
<p>Those configuration, being standard by other plugin manager are easy to copy,
edit and understand just following each plugin README.</p>
<p>Lazy goes another route where it needs specific syntax to install and configure
a plugin. This is a requirement for lazy loading. This allows lazy-nvim to hook
on specific events and keymaps to launch a plugin.</p>
<p>The same package, configured under lazy.nvim will more look like this:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="lua"><span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">require</span><span>(</span><span style="color: #A6D189;">"lazy.nvim"</span><span>).</span><span style="color: #8CAAEE;font-style: italic;">setup</span><span>({</span></span>
<span class="giallo-l"><span> {</span></span>
<span class="giallo-l"><span style="color: #A6D189;"> "mbbill/undotree"</span><span>,</span></span>
<span class="giallo-l"><span> lazy</span><span style="color: #81C8BE;"> =</span><span style="color: #E78284;"> true</span><span>,</span></span>
<span class="giallo-l"><span> keys</span><span style="color: #81C8BE;"> =</span><span> {</span></span>
<span class="giallo-l"><span> { </span><span style="color: #A6D189;">"<leader>u"</span><span>, </span><span style="color: #CA9EE6;">function</span><span style="color: #949CBB;">()</span><span> vim.cmd.</span><span style="color: #8CAAEE;font-style: italic;">UndotreeToggle</span><span>() </span><span style="color: #CA9EE6;">end</span><span>,</span></span>
<span class="giallo-l"><span> desc</span><span style="color: #81C8BE;"> =</span><span style="color: #A6D189;"> "Show nvim undotree"</span><span>,</span></span>
<span class="giallo-l"><span> },</span></span>
<span class="giallo-l"><span> config</span><span style="color: #81C8BE;"> =</span><span style="color: #E78284;"> true</span><span>,</span></span>
<span class="giallo-l"><span> },</span></span>
<span class="giallo-l"><span> }</span></span>
<span class="giallo-l"><span>})</span></span></code></pre>
<p>Notice the specific syntax to setup new keybindings, it allows lazy-nvim to
bind those keys to the loading of the plugin, thus supporting lazy loading.</p>
<p>It highlights a couple of struggles I had with lazy-nvim:</p>
<ol>
<li>Documentation is sparse while requiring a mental model shift</li>
<li>This syntax being non-standard does result in a ecosystem split</li>
</ol>
<p>At least while I become familiar with lazy-nvim, this causes some difficulties
when configuring unknown plugins where I need to understanding
the specifics of a plugin and them map this to the proper lazy-nvim syntax.
This is similar to the layered configuration issue Nix packages have.</p>
<p>Similarly, during first setup it was not very clear from the documentation the
difference between <code>init</code>, <code>config</code>, <code>setup</code>, <code>opts</code> & <code>main</code>. I should
probably open a pull request to add some concrete examples, this would help the
understanding.</p>
<h2 id="conclusion">Conclusion</h2>
<p>I am keeping Lazy-nvim for now, it is a great package manager. I am a bit
afraid of the day I'll need to move out of it, given it's configuration
lock-in. I haven't tested the auto-reload feature which seems very interesting.</p>
<p>You can see my nvim configuration at: <a rel="external" href="https://github.com/dimtion/dotfiles/tree/master/.config/nvim">dimtion/dotfiles</a></p>
The magic of no maintenance2023-07-24T00:00:00+00:002023-07-24T00:00:00+00:00
Unknown
https://dimtion.fr/blog/the-magic-of-no-maintenance/<p>Recently, I've been revisiting some of my old side projects. I'm in the process
of consolidating all those projects onto a single server. This would help with
cost of managing all of them, and also merge all the Apache2, nginx, Traefik,
uwsgi, actix-web, and whatnot web servers configurations I tried over the last
10 years. This project is quite satisfying and feels like a form of
introspection on the different things and technologies I tried in my free time
over the years.</p>
<p>I thank my past self for simplifying projects on-boarding: most projects do
have a Makefile and a <code>deploy.sh</code> file to manage build, run, and deployment. It
makes wrapping one's head around a forgotten project straightforward.
Most of those projects only had a single developer working on them, though, it
remains valuable to have a consistent entrypoint to spin up a polyglot tech
stack.</p>
<hr />
<p>There is one hurdle I encountered though, independently of what package manager
I would use, nearly none of the projects I worked on builds or runs on the
first try. There would always be a dependency that is no longer available or
incompatible with the toolchain I have on my computer in 2023. And then,
upgrading that dependency becomes a game of whack-a-mole between upgrading to a
recent-enough version, but not needing to upgrade peer-dependencies that are now
no longer compatible with those breaking changes.</p>
<p>In that context, <a rel="external" href="https://semver.org/">semver</a> is of no help. Anything can be a
<a rel="external" href="https://xkcd.com/1172/">breaking change</a>.</p>
<p>Vagrant, Cargo, requirement.txt, Opam, Dockerfile, npm, yarn, all would fail in
one way or another.</p>
<p>A couple of projects did build and run on the first try, and the satisfaction of
seeing the programs run without a hassle was kind of similar to doing a strike
at bowling on the first try!</p>
<hr />
<p>There is one exception though, and it would be for Go projects. If anything Go
got right for long-term maintainability, it would be dependency management.
Contrary to other dependency managers, go-pkg downloads the source code locally
and stores it on disk. Come back 1, 5 or 10 years later a go project will
build (assuming that the compiler remains the same).</p>
<p>There is a tension though, go-pkg is not the only package manager to download
the dependencies locally (all of them do it one way or another). However, given
the size of those dependencies in other languages (e.g. <code>node_modules</code>) go is
one of the only languages where it is considered good practice to checkout the
vendors to the VCS.</p>
<p>This might be utopian, but the take-out is that for small side projects to
remain maintainable over the years, strive to have all the code checked-out in
the VCS, and keep them small.</p>
Why Large Language Models Won't Fix Google2023-04-02T00:00:00+00:002023-04-02T00:00:00+00:00
Unknown
https://dimtion.fr/blog/llm-google/<p>It is a <a rel="external" href="https://news.ycombinator.com/item?id=30347719">common</a> <a rel="external" href="https://news.ycombinator.com/item?id=29772136">meme</a> <a rel="external" href="https://news.ycombinator.com/item?id=29392702">that</a> <a rel="external" href="https://news.ycombinator.com/item?id=30946792">Google</a> <a rel="external" href="https://news.ycombinator.com/item?id=32201420">Search</a> results quality has declined through the years.
Despite being sold by many as such, Large Language Models
(LLMs) such as ChatGPT, Bing Chat and Bard are not on a course to improve
search engines. This is not just a technical issue, but a fundamental
misunderstanding of the environment in which search engines operate.</p>
<p>Let's tackle the obvious technical limitation right away: LLMs are just
statistical token predictors. Even though they do exhibit <a rel="external" href="https://arxiv.org/abs/2303.12712">emergent properties</a>,
it is crucial to understand that their objective function is not to achieve a
reasonable truth, but rather to <a rel="external" href="https://en.wikipedia.org/wiki/Autoregressive_model">predict</a> the most probable token in a sequence
based on their embedded space. In other words, if the dataset on which the
model is trained is not more <em>accurate</em> than the index of named search engine, no
valuable gain can be expected.</p>
<hr />
<p>Google, being the most used search engine ever, is currently the most exposed
it ever has been to <a rel="external" href="https://twitter.com/0xgaut/status/1641619593830514688">SEO manipulation</a>. It is slowly losing the war on organic
content despite billions spent by the company on counter measures.</p>
<p>Today, ChatGPT's exceptional ability to produce organic responses to simple
requests, demonstrates that there might be a door to a world where algorithmic
search is a problem under control. Despite
tendencies to hallucinate <a rel="external" href="https://archive.ph/dDjke">obviously</a> or <a rel="external" href="https://mashable.com/article/google-bard-james-webb-telescope-false-fact">non-obviously</a>
responses, those seem more related to the fact that data is missing from the
training set than because of malice.</p>
<p>This fundamentally is incorrect. ChatGPT outputs seemingly looking more organic is only due
to the fact that SEO adversarial actors have not yet caught up on how to
exploit the training, fine tuning and prompt engineering powering the
tech. Especially not at <a rel="external" href="https://blog.searchmysite.net/posts/almost-all-searches-on-my-independent-search-engine-are-now-from-seo-spam-bots/">industrial levels</a>
at which Google is fighting against.</p>
<p>Alas, our short internet memory already forgot why page-rank was considered as
revolutionary when Google was first introduced to the world: it was much more
difficult to deceive than competitive search engines.</p>
<p>In a world where a ChatGPT becomes the most common gateway to access the web,
adversarial actors will quickly catch up on its limits, and we'll be back to
the good ol' mediocre results.</p>
<hr />
<p>The more important reason why search engine accuracy has declined is due the
pressure to generate profits. Google, as a publicly traded company, derives
most of its revenue from advertising. This means that there is a financial
incentive to prioritize ads over organic search results. This has resulted over
the years in more ads appearing in search results, which can push down organic
results.</p>
<p>LLMs, however, do not face the same pressure to generate profits. At least not
yet. They are still seen as a technology that is being developed and improved,
and as such, they are not under the same financial pressure as traditional
search engines. However, this is likely to change once LLMs become more widely
adopted. At that point, they will face the same pressure to generate profits as
traditional search engines. This means that LLMs will likely start to
prioritize ads over organic response, just like traditional search engines.</p>
<p>Appendix: This is only the beginning: <a rel="external" href="https://twitter.com/debarghya_das/status/1640892791923572737">https://twitter.com/debarghya_das/status/1640892791923572737</a></p>
<!--
In conclusion I'd like to leave the last word to ChatGPT:
> Large language models are not the silver bullet for fixing
> search engines. While they may improve search accuracy in the short term, they
> will not be able to do so in the long run. This is because LLMs are designed to
> assist search engines, not replace them. They are an add-on tool that can help
> improve search results, but they cannot replace the search algorithm entirely.
> Furthermore, as we have seen, LLMs are not immune to the same problems that
> traditional search engines face, such as SEO optimization and profitability.
> Therefore, we should not rely solely on LLMs to fix search engines. Instead, we
> need to continue to develop and improve the search algorithms themselves, while
> also using LLMs as a complementary tool.
## Scratchpad
TODO:
- [ ] Even if LLM search could magically be orders of magnitude cheaper than
Page-rank like, this will not solve the profitability issue
- [ ] There is hope: there could be a paradism change in a world of
generative ai
- [ ] Don't forget that a lot of LLM usage is backing the prompt or fine-tune
with "old-school" search engine
-->
Hold your left2022-02-03T00:00:00+00:002022-02-03T00:00:00+00:00
Unknown
https://dimtion.fr/blog/tips-invert-conditions/<p><em>Error handling is hard</em>. When writing <a rel="external" href="https://en.wikipedia.org/wiki/Machine_learning">throwable
code</a>, and requirements are unmet, developers generally
glance over error handling and let the script crash. However, when writing
<a rel="external" href="https://www.reddit.com/r/ProgrammerHumor/comments/gf4ujl/production_ready/">production-ready</a>
code, error handling oftentimes makes up the bulk of the source code. This has
a non-trivial impact on readability, maintainability, and debugging.</p>
<p>In this article, we'll explore two very simple tricks that will help make
error-handling a breathe.</p>
<h2 id="a-first-example">A first example</h2>
<p>Consider the following <code>update_item</code> function:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="python"><span class="giallo-l"><span>VALID_STATUS</span><span style="color: #81C8BE;"> =</span><span style="color: #949CBB;"> {</span><span style="color: #A6D189;"> "ON"</span><span style="color: #949CBB;">,</span><span style="color: #A6D189;"> "OFF"</span><span style="color: #949CBB;"> }</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;">def</span><span style="color: #8CAAEE;font-style: italic;"> update_item</span><span style="color: #949CBB;">(</span><span style="color: #EA999C;font-style: italic;">tenant</span><span style="color: #949CBB;">:</span><span style="color: #EA999C;"> User</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> item_id</span><span style="color: #949CBB;">:</span><span style="color: #CA9EE6;font-style: italic;"> str</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> new_status</span><span style="color: #949CBB;">:</span><span style="color: #CA9EE6;font-style: italic;"> str</span><span style="color: #949CBB;">) -></span><span> Item</span></span>
<span class="giallo-l"><span style="color: #A6D189;"> """Update an item owned by a tenant with a new status"""</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> tenant</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">is_authorized</span><span style="color: #949CBB;">():</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> try</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span> item</span><span style="color: #81C8BE;"> =</span><span> db</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">fetch_item</span><span style="color: #949CBB;">(</span><span>item_id</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> !item</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">is_locked</span><span style="color: #949CBB;">():</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> new_status</span><span style="color: #CA9EE6;"> in</span><span> VALID_STATUS</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> item</span><span style="color: #949CBB;">.</span><span>owner</span><span style="color: #949CBB;">.</span><span>id</span><span style="color: #81C8BE;"> ==</span><span> tenant</span><span style="color: #949CBB;">.</span><span>id</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span> item</span><span style="color: #949CBB;">.</span><span>status</span><span style="color: #81C8BE;"> =</span><span> new_status</span></span>
<span class="giallo-l"><span> item</span><span style="color: #81C8BE;"> =</span><span> db</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">persist_item</span><span style="color: #949CBB;">(</span><span>item</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> return</span><span> item</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> else</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;">"Tenant does not own item"</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> else</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> ValueError</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Invalid status '</span><span style="color: #F4B8E4;">{</span><span>new_status</span><span style="color: #F4B8E4;">}</span><span style="color: #A6D189;">'"</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> else</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;">"Cannot update a locked item"</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> except</span><span> UnknownItemException</span><span style="color: #CA9EE6;"> as</span><span> e</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Cannot find item '</span><span style="color: #F4B8E4;">{</span><span>item_id</span><span style="color: #F4B8E4;">}</span><span style="color: #A6D189;">'"</span><span style="color: #949CBB;">)</span><span style="color: #CA9EE6;"> from</span><span> e</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> except</span><span style="color: #EF9F76;font-style: italic;"> ConnectionError</span><span style="color: #CA9EE6;"> as</span><span> e</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Error connecting to the database"</span><span style="color: #949CBB;">)</span><span style="color: #CA9EE6;"> from</span><span> e</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> else</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Owner is not authorized to update statuses"</span><span style="color: #949CBB;">)</span></span></code></pre>
<p>This code looks complicated, however, it does something simple: given a set of
business rules and user inputs, it fetches an <code>item</code>, check that the input is
valid and the conditions are met, and then update and persist the item.</p>
<p>To help refactor this code, we'll introduce a set of rules:</p>
<ul>
<li><strong>Rule 1: Hold your left</strong></li>
<li><strong>Rule 2: Happy path last comes last</strong></li>
</ul>
<p>Let's jump to them and explore some examples.</p>
<h2 id="rule-1-hold-your-left">Rule 1: Hold your left</h2>
<p>This rule boils down to: when entering a conditional block, you should write as
few code as possible in the nested block, and try to bail out of the block as
soon as possible.</p>
<p>For example, instead of writing:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="python"><span class="giallo-l"><span style="color: #CA9EE6;">def</span><span style="color: #8CAAEE;font-style: italic;"> main</span><span style="color: #949CBB;">():</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> business_rule</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;"> execute_routine_1</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;"> execute_routine_2</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;"> execute_routine_3</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;"> execute_routine_4</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;"> execute_routine_5</span><span style="color: #949CBB;">()</span></span></code></pre>
<p>Write:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="python"><span class="giallo-l"><span style="color: #CA9EE6;">def</span><span style="color: #8CAAEE;font-style: italic;"> main</span><span style="color: #949CBB;">():</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> !business_rule</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> return</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #8CAAEE;"> execute_routine_1</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;"> execute_routine_2</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;"> execute_routine_3</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;"> execute_routine_4</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;"> execute_routine_5</span><span style="color: #949CBB;">()</span></span></code></pre>
<p>Here the bulk of the code is located outside of the nested block. Making the
average line length shorter.</p>
<p>Following this rule will help remove nested blocks that are hard to follow.
Each nested block is a new state to keep in mind, which makes reasoning about
the code more complicated.</p>
<p>Having most of the code outside of a nested block will help the maintainer or
reviewer to understand what the code is actually about.</p>
<p>This rule alone is not enough to make our code more maintainable, we need to
combine it with rule 2.</p>
<h2 id="2-happy-path-comes-last">2. Happy path comes last</h2>
<p>This rule is two-fold: following it will help implement code that respects
rule one. But more importantly, this rule improves code quality by making the
developer stop and think about edge cases first. The assumption being: you are
very unlikely to forget to implement the happy path, this is what stakeholders
will look at first, however, edge cases could be forgotten if their
implementation is pushed at the bottom of a function.</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="python"><span class="giallo-l"><span style="color: #CA9EE6;">def</span><span style="color: #8CAAEE;font-style: italic;"> push_ball</span><span style="color: #949CBB;">(</span><span style="color: #EA999C;font-style: italic;">ball</span><span style="color: #949CBB;">:</span><span style="color: #EA999C;"> Ball</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> speed</span><span style="color: #949CBB;">:</span><span style="color: #EA999C;"> number</span><span style="color: #949CBB;">):</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> speed</span><span style="color: #81C8BE;"> ></span><span style="color: #EF9F76;"> 0</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span> ball</span><span style="color: #949CBB;">.</span><span>speed</span><span style="color: #81C8BE;"> +=</span><span> speed</span></span>
<span class="giallo-l"><span> ball</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">notify</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> else</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;">"Speed must be positive"</span><span style="color: #949CBB;">)</span></span></code></pre>
<p>Following rule 1, we could refactor this code:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="python"><span class="giallo-l"><span style="color: #CA9EE6;">def</span><span style="color: #8CAAEE;font-style: italic;"> push_ball</span><span style="color: #949CBB;">(</span><span style="color: #EA999C;font-style: italic;">ball</span><span style="color: #949CBB;">:</span><span style="color: #EA999C;"> Ball</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> speed</span><span style="color: #949CBB;">:</span><span style="color: #EA999C;"> number</span><span style="color: #949CBB;">):</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> speed</span><span style="color: #81C8BE;"> ></span><span style="color: #EF9F76;"> 0</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span> ball</span><span style="color: #949CBB;">.</span><span>speed</span><span style="color: #81C8BE;"> +=</span><span> speed</span></span>
<span class="giallo-l"><span> ball</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">notify</span><span style="color: #949CBB;">()</span></span>
<span class="giallo-l" style="background-color: #C6D0F512;"><span style="color: #81C8BE;">+</span><span style="color: #CA9EE6;"> return</span></span>
<span class="giallo-l" style="background-color: #C6D0F512;"><span style="color: #81C8BE;">+</span></span>
<span class="giallo-l" style="background-color: #C6D0F512;"><span style="color: #81C8BE;">~</span><span style="color: #949CBB;font-style: italic;"> # Unecessary else, hold your left!</span></span>
<span class="giallo-l" style="background-color: #C6D0F512;"><span style="color: #81C8BE;">~</span><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;">"Speed must be positive"</span><span style="color: #949CBB;">)</span></span></code></pre>
<p>With rule 1 & 2 in mind, we can then invert the if condition, to push the happy
path at the bottom of our code:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="python"><span class="giallo-l"><span style="color: #CA9EE6;">def</span><span style="color: #8CAAEE;font-style: italic;"> push_ball</span><span style="color: #949CBB;">(</span><span style="color: #EA999C;font-style: italic;">ball</span><span style="color: #949CBB;">:</span><span style="color: #EA999C;"> Ball</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> speed</span><span style="color: #949CBB;">:</span><span style="color: #EA999C;"> number</span><span style="color: #949CBB;">):</span></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> # Check for the edge case first:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> speed</span><span style="color: #81C8BE;"> <=</span><span style="color: #EF9F76;"> 0</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;">"Speed must be positive"</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span> ball</span><span style="color: #949CBB;">.</span><span>speed</span><span style="color: #81C8BE;"> +=</span><span> speed</span></span>
<span class="giallo-l"><span> ball</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">notify</span><span style="color: #949CBB;">()</span></span></code></pre>
<p>With the happy path on the left-most column, it becomes obvious where to look
for the main action of a function: <strong>bottom-left of any code block</strong>. This can seem
counter-intuitive since the visual cortex generally scans text from top-left to
bottom-right, but you'll notice that functions following rules 1 and 2 won't
have a lot of nested code: the happy path will always be easy to find when you
know where to look for it.</p>
<h2 id="let-s-rewrite-our-first-example">Let's rewrite our first example</h2>
<p>With those rules in mind, let's rewrite our original function:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="python"><span class="giallo-l"><span>VALID_STATUS</span><span style="color: #81C8BE;"> =</span><span style="color: #949CBB;"> {</span><span style="color: #A6D189;"> "ON"</span><span style="color: #949CBB;">,</span><span style="color: #A6D189;"> "OFF"</span><span style="color: #949CBB;"> }</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;">def</span><span style="color: #8CAAEE;font-style: italic;"> update_item</span><span style="color: #949CBB;">(</span><span style="color: #EA999C;font-style: italic;">tenant</span><span style="color: #949CBB;">:</span><span style="color: #EA999C;"> User</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> item_id</span><span style="color: #949CBB;">:</span><span style="color: #CA9EE6;font-style: italic;"> str</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> new_status</span><span style="color: #949CBB;">:</span><span style="color: #CA9EE6;font-style: italic;"> str</span><span style="color: #949CBB;">) -></span><span> Item</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #A6D189;"> """Update an item owned by a tenant with a new status"""</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> # Execute validation linearly:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> !tenant</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">is_authorized</span><span style="color: #949CBB;">():</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Owner is not authorized to update statuses"</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span> item</span><span style="color: #949CBB;">:</span><span> Item</span><span style="color: #81C8BE;"> =</span><span style="color: #EF9F76;"> None</span><span style="color: #949CBB;font-style: italic;"> # Declare item in the outerscope</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> try</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span> item</span><span style="color: #81C8BE;"> =</span><span> db</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">fetch_item</span><span style="color: #949CBB;">(</span><span>item_id</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> except</span><span> UnknownItemException</span><span style="color: #CA9EE6;"> as</span><span> e</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Cannot find item '</span><span style="color: #F4B8E4;">{</span><span>item_id</span><span style="color: #F4B8E4;">}</span><span style="color: #A6D189;">'"</span><span style="color: #949CBB;">)</span><span style="color: #CA9EE6;"> from</span><span> e</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> except</span><span style="color: #EF9F76;font-style: italic;"> ConnectionError</span><span style="color: #CA9EE6;"> as</span><span> e</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Error connecting to the database"</span><span style="color: #949CBB;">)</span><span style="color: #CA9EE6;"> from</span><span> e</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> item</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">is_locked</span><span style="color: #949CBB;">():</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;">"Cannot update a locked item"</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> new_status</span><span style="color: #CA9EE6;"> not in</span><span> VALID_STATUS</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> ValueError</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Invalid status '</span><span style="color: #F4B8E4;">{</span><span>new_status</span><span style="color: #F4B8E4;">}</span><span style="color: #A6D189;">'"</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> item</span><span style="color: #949CBB;">.</span><span>owner</span><span style="color: #949CBB;">.</span><span>id</span><span style="color: #81C8BE;"> !=</span><span> tenant</span><span style="color: #949CBB;">.</span><span>id</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Tenant does not own item '</span><span style="color: #F4B8E4;">{</span><span>item_id</span><span style="color: #F4B8E4;">}</span><span style="color: #A6D189;">'"</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> # Finally run happy path:</span></span>
<span class="giallo-l"><span> item</span><span style="color: #949CBB;">.</span><span>status</span><span style="color: #81C8BE;"> =</span><span> new_status</span></span>
<span class="giallo-l"><span> item</span><span style="color: #81C8BE;"> =</span><span> db</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">persist_item</span><span style="color: #949CBB;">(</span><span>item</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> return</span><span> item</span></span></code></pre>
<p>You'll notice that:</p>
<ul>
<li>The code is vertically longer, but horizontally narrower</li>
<li>Even if the code is longer, it is easier to follow: no nested conditions, can
be read in a linear fashion</li>
<li>The happy path is literally the last lines of the block, at the bottom of function</li>
<li>Conditions that are related are visually grouped using vertical <a rel="external" href="https://wiki.c2.com/?SyntacticallySignificantWhitespaceConsideredHarmful">white space
management</a></li>
</ul>
<h2 id="a-mental-model-fence-oriented-programming">A mental model: fence oriented programming</h2>
<!--
<img src="https://dimtion.fr/processed_images/interactive-sports-VYTQNnaboUA-unsplash.1443b96b00d1c041.jpg" />
-->
<p>One way to mentally visualize this coding pattern is to view each <code>if</code>
statement as a
<a rel="external" href="https://en.wikipedia.org/wiki/G._K._Chesterton#Chesterton's_fence">fence</a> (or
guard) that needs to be passed before continuing execution. Each fence insure
that a requirement is met before executing the
happy path. Adding or removing a requirement is simply adding or removing a new
fence.</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="plain"><span class="giallo-l"><span> FENCE FENCE</span></span>
<span class="giallo-l"><span>| | / || / || / \</span></span>
<span class="giallo-l"><span>| start | -> | condition A || -- yes -> | condition B || -- yes -> | Execute action |</span></span>
<span class="giallo-l"><span>| | \ met? || \ met? || \ /</span></span>
<span class="giallo-l"><span> | |</span></span>
<span class="giallo-l"><span> -- no -> | abort | -- no -> | abort |</span></span></code></pre>
<p>This mental model is similar to <a rel="external" href="https://fsharpforfunandprofit.com/rop/">Railway Oriented
Programming</a> in functional
programming languages.</p>
<p>By following the two above rules, reading a piece of code the feels just like
reading a checklist of conditions to met before executing the happy path.</p>
<h3 id="update-item-checklist"><code>update_item</code> checklist</h3>
<p>To learn fence oriented programming, one way is to write our code in a form of a checklist:</p>
<ul>
<li>☑️ Is <code>tenant</code> authorized to update statuses?</li>
<li>☑️ Is the <code>item</code> present in the database?</li>
<li>☑️ Was reading from the database successful?</li>
<li>☑️ Is the item unlocked?</li>
<li>☑️ Is the input status a valid status?</li>
<li>☑️ Is the tenant the owner of the item?</li>
</ul>
<p>Writing the business logic as a checklist makes writing the code trivial then,
just a set of instructions to follow!</p>
<h2 id="bonus-benefit-trivial-conditional-reordering">Bonus benefit: trivial conditional reordering</h2>
<p>Some of you might have noticed, there is a minor bug in the code example
presented: <code>status</code> validation is executed after item retrieval, even if the item
is not needed to check the validity of the input. Depending on the database
technology and the service architecture, this could become a potential attack
vector against our service. For example, an attacker could execute a large
number of queries against our service with invalid statuses, hammering the
database with unnecessary queries.</p>
<p>Ignoring <a rel="external" href="https://en.wikipedia.org/wiki/Timing_attack">timing-based side-channel
attacks</a>, a simple fix would be to
reorder the checks by first doing the status validation, then fetch the item in
the database.</p>
<p>Using the inverted condition pattern, this reordering is as simple as moving
lines around in our function. while with our first example, doing such would
be a dance of moving the correct <code>if/else</code> condition and correctly matching
indentation...</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="python"><span class="giallo-l"><span>VALID_STATUS</span><span style="color: #81C8BE;"> =</span><span style="color: #949CBB;"> {</span><span style="color: #A6D189;"> "ON"</span><span style="color: #949CBB;">,</span><span style="color: #A6D189;"> "OFF"</span><span style="color: #949CBB;"> }</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;">def</span><span style="color: #8CAAEE;font-style: italic;"> update_item</span><span style="color: #949CBB;">(</span><span style="color: #EA999C;font-style: italic;">tenant</span><span style="color: #949CBB;">:</span><span style="color: #EA999C;"> User</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> item_id</span><span style="color: #949CBB;">:</span><span style="color: #CA9EE6;font-style: italic;"> str</span><span style="color: #949CBB;">,</span><span style="color: #EA999C;font-style: italic;"> new_status</span><span style="color: #949CBB;">:</span><span style="color: #CA9EE6;font-style: italic;"> str</span><span style="color: #949CBB;">) -></span><span> Item</span></span>
<span class="giallo-l"><span style="color: #A6D189;"> """Update an item owned by a tenant with a new status"""</span></span>
<span class="giallo-l"></span>
<span class="giallo-l" style="background-color: #C6D0F512;"><span style="color: #81C8BE;">+</span><span style="color: #949CBB;font-style: italic;"> # Executing input validation before authorization and consistency checks:</span></span>
<span class="giallo-l" style="background-color: #C6D0F512;"><span style="color: #81C8BE;">+</span><span style="color: #CA9EE6;"> if</span><span> new_status</span><span style="color: #CA9EE6;"> not in</span><span> VALID_STATUS</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l" style="background-color: #C6D0F512;"><span style="color: #81C8BE;">+</span><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> ValueError</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Invalid status '</span><span style="color: #F4B8E4;">{</span><span>new_status</span><span style="color: #F4B8E4;">}</span><span style="color: #A6D189;">'"</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> # Execute validation linearly:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> !tenant</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">is_authorized</span><span style="color: #949CBB;">():</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Owner is not authorized to update statuses"</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span> item</span><span style="color: #949CBB;">:</span><span> Item</span><span style="color: #81C8BE;"> =</span><span style="color: #EF9F76;"> None</span><span style="color: #949CBB;font-style: italic;"> # Declare item in the outerscope</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> try</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span> item</span><span style="color: #81C8BE;"> =</span><span> db</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">fetch_item</span><span style="color: #949CBB;">(</span><span>item_id</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> except</span><span> UnknownItemException</span><span style="color: #CA9EE6;"> as</span><span> e</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Cannot find item '</span><span style="color: #F4B8E4;">{</span><span>item_id</span><span style="color: #F4B8E4;">}</span><span style="color: #A6D189;">'"</span><span style="color: #949CBB;">)</span><span style="color: #CA9EE6;"> from</span><span> e</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> except</span><span style="color: #EF9F76;font-style: italic;"> ConnectionError</span><span style="color: #CA9EE6;"> as</span><span> e</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Error connecting to the database"</span><span style="color: #949CBB;">)</span><span style="color: #CA9EE6;"> from</span><span> e</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> item</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">is_locked</span><span style="color: #949CBB;">():</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;">"Cannot update a locked item"</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l" style="background-color: #C6D0F512;"><span style="color: #81C8BE;">-</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> if</span><span> item</span><span style="color: #949CBB;">.</span><span>owner</span><span style="color: #949CBB;">.</span><span>id</span><span style="color: #81C8BE;"> !=</span><span> tenant</span><span style="color: #949CBB;">.</span><span>id</span><span style="color: #949CBB;">:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> raise</span><span style="color: #EF9F76;font-style: italic;"> Exception</span><span style="color: #949CBB;">(</span><span style="color: #A6D189;font-style: italic;">f</span><span style="color: #A6D189;">"Tenant does not own item '</span><span style="color: #F4B8E4;">{</span><span>item_id</span><span style="color: #F4B8E4;">}</span><span style="color: #A6D189;">'"</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"> # Finally run happy path:</span></span>
<span class="giallo-l"><span> item</span><span style="color: #949CBB;">.</span><span>status</span><span style="color: #81C8BE;"> =</span><span> new_status</span></span>
<span class="giallo-l"><span> item</span><span style="color: #81C8BE;"> =</span><span> db</span><span style="color: #949CBB;">.</span><span style="color: #8CAAEE;">persist_item</span><span style="color: #949CBB;">(</span><span>item</span><span style="color: #949CBB;">)</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;"> return</span><span> item</span></span></code></pre>Using Git Efficiently2021-01-05T00:00:00+00:002021-01-05T00:00:00+00:00
Unknown
https://dimtion.fr/blog/average-engineer-tips/<h2 id="introduction">Introduction</h2>
<p>There was a debate a few months back on Twitter about <a rel="external" href="https://twitter.com/dmartincy/status/1150156226761830400">10x
developers</a>, if they
exist, what is the definition of such unicorn, or even if 10x developers are
only bad developers who simply look good to management.</p>
<p>The truth is, without any prior knowledge, if we assume that <em>effectiveness</em>
can be measured (whatever definition you want to put in that word), this
distribution will most likely follow a <a rel="external" href="https://en.wikipedia.org/wiki/Normal_distribution">bell
curve</a> over engineers.
You the reader, and I the writer, are mathematically most likely to be in the
middle of the curve. In other words: we are average developers.</p>
<p>If I, an average developer, want to get more <em>effective</em>, instead of only
looking up at highly effective developers, we need to understand the
characteristics of an average developer. This allows us to overcome the
<a rel="external" href="https://en.wikipedia.org/wiki/Survivorship_bias">survivorship bias</a> when
fishing for advice.</p>
<p>What would an average developer look like? Depending on your definition of
<em>effective</em>, it could mean:</p>
<ul>
<li>
<p>A developer that regularly makes mistakes, probably mostly stupid errors:
forgetting to push a file, adding a typo when fixing another typo, forgetting
to refresh the Github page when going through the review comments</p>
</li>
<li>
<p>A developer that delivers average code: code that does the job but no more
no less, code that doesn't handle all error cases, code that is readable but
not necessarily straightforward</p>
</li>
<li>
<p>A developer that reads, reviews, and edits code written by other average
developers: code that should have been refactored 10 years ago, that was
designed in a rush for its initial use case, and is now stretched out to
its inflection point</p>
</li>
<li>
<p>A developer that works in an average company that doesn't follow every
best practice, a company that needs every new feature for yesterday</p>
</li>
</ul>
<p>If you recognize yourself in one of those bullet points, congratulation, you
are an average developer! (Spoiler: everybody is average, just select your
metric accordingly).</p>
<p>That said, there are heuristics you can follow to look less average within
your peers, these tips won't make you make fewer mistakes, or make you smarter
<em>per se</em>, but following them will help you introspect your biases and become a
better engineer.</p>
<p>I've tried to make them as practical as possible.</p>
<h2 id="part-1-using-git-effectively">Part 1: Using git effectively</h2>
<p>This is probably the most important advice, doing pair-coding with junior
developers, I've seen so many instances of the following anti-pattern:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="plain"><span class="giallo-l"><span><change a file></span></span>
<span class="giallo-l"><span>:wq <quit text editor></span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span>git add -A</span></span>
<span class="giallo-l"><span>git commit -m "Fix typo"</span></span>
<span class="giallo-l"><span>git push</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span><Click on re-request review on github></span></span></code></pre>
<p>This is probably the worse set of commands you can run, 95% percent of the time
you will look fast and swift, but in the 5 other percent, you will look like a
fool in front of your colleague reviewing your pull request.</p>
<p>Let's break it down to a more careful and methodic way to create new commits.</p>
<h3 id="tip-1-proofread-when-stashing-changes">Tip 1: Proofread when stashing changes</h3>
<p>The first advice is to never, <strong>ever</strong>, use <code>git add -A</code>. With <code>-A</code> parameter
you never know what you are committing, you could still have a debug statement
remaining, forgot to clean up the last issue you were working on, or simply
staging files you don't mean to commit.</p>
<p>When you create a commit, not only make sure to know exactly which file you are
committing, but also make sure you have read each line before staging them. A
great tool to use is git with <code>git add -p</code> or <code>git add --patch</code> in its long-form.</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># Prefer patch mode over --all:</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">git</span><span style="color: #A6D189;"> add -p</span></span></code></pre>
<p>The <code>-p</code> parameter allows you to visualize each patch of a commit you are going
to create. You'll know exactly which change you are going to commit. Make
sure you are reading every single line of each patch!</p>
<p>In particular, before adding a patch, look carefully for:</p>
<ul>
<li>Typos in variable names</li>
<li>Remaining debug statements</li>
<li><code>TODO</code> statements</li>
<li>Read the surrounding comments and docstrings: are they still relevant and
correct with the changes you are about to commit?</li>
</ul>
<p>Once you have staged your changes, don't hesitate to have a quick glance at the
changes you made by running:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># Changes staged for commit:</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">git</span><span style="color: #A6D189;"> diff --staged</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># Changes not staged for commit:</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">git</span><span style="color: #A6D189;"> diff</span></span></code></pre>
<p>Make sure that you did not forget any new file in the staging area!</p>
<h3 id="tip-2-force-yourself-to-write-effective-git-messages">Tip 2: force yourself to write effective git messages</h3>
<p>Now that you have reviewed the changes you made, you are ready to create a new
commit. Here again, I advise against using any shortcut like <code>git commit -m</code>.
Even if <code>-m</code> is swift, it encourages the developer to write unhelpful commit
messages.</p>
<p>Prefer to use <code>git commit</code> in verbose mode:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># Always use verbose mode:</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">git</span><span style="color: #A6D189;"> commit -v</span></span></code></pre>
<p>When writing a meaningful commit message, you will be able to see the full
change that will be created. This serves two functions:</p>
<ol>
<li>Make sure again at a glance that you are committing what you want to commit.
You don't need to read every line, that was done during staging, but have a
glance at the patch and trust your subconscious mind to spot any irregularity</li>
<li>When writing a commit message, you now have under your eyes the changes you
are about to commit. This is a great way to write a relevant commit message.</li>
</ol>
<p>There are already a ton of articles on how to write a great commit message, so I
won't spend too much time on that. Basically, those good ideas can be summarized
to:</p>
<ul>
<li>Be consistent in the formatting, if possible have <a rel="external" href="https://www.conventionalcommits.org/en/v1.0.0/">a
convention</a> with your team on
how commit messages should be formated</li>
<li>Have a commit title that explains <em>what</em> has been changed</li>
<li>When necessary have a commit body that explains <em>why</em> those changes have been made</li>
<li>Have in the title or the body of the message a reference to your project bug tracker</li>
</ul>
<p>When creating the commit message, especially for trivial changes, resist the
urge to set an irrelevant commit title like:</p>
<ul>
<li><code>fix typo</code></li>
<li><code>pr changes</code></li>
<li><code>review ~1</code></li>
<li><code>review ~2</code></li>
</ul>
<p>You don't need to have a long commit message with a full body for each change,
especially if the patch is trivial. Try to force yourself to always write a
relevant commit message. This might take 5 seconds more than simply writing
<code>fix</code>, but this will save countless time when another teammate will find the
source of a bug in a tree of 10 000 commits.</p>
<p>Here are examples of better commit titles for trivial changes:</p>
<ul>
<li><code>scheduler: typo rename actionHander to actionHandler</code></li>
<li><code>PROJ-12031 core: simplify deduplicate_str() implementation</code></li>
</ul>
<h3 id="tip-3-never-ever-ever-use-push-force">Tip 3: Never, ever, ever, use push --force`</h3>
<p>Your commit is now created locally. You can now push your changes with <code>git push</code> or <code>git push -u origin <branch-name></code> if you are creating a new branch.</p>
<p>If you really need to push force (after a <code>git commit -v --amend</code> or a <code>git rebase</code> for example) prefer <code>git push --force-with-lease</code> over <code>git push --force</code>.</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># Amend some changes to an existing commit:</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">git</span><span style="color: #A6D189;"> commit -v --amend</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># Never push force:</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">git</span><span style="color: #A6D189;"> push --force-with-lease</span></span></code></pre>
<p>with <code>force-with-lease</code>, if the remote branch has been modified from you last fetch,
git will not override the changes. This is a great failsafe to avoid
overwriting one of your colleagues' work.</p>
<h3 id="tip-4-proof-read-again-and-again">Tip 4: Proof read again, and again</h3>
<p>If your team is using Github, when creating the Pull Request don't hesitate to
give a last glance at the diff, seeing your change in another context than your
text editor is a great way to catch last minutes typos.</p>
<h2 id="automate-your-git-workflow">Automate your git workflow</h2>
<p>Depending on which language you are developing in, the compiler might not be
bundled with static analysis that could catch common pitfalls. For those
languages, you might already rely on a linter on top of your test suite.</p>
<p>Linters not only serve the role of catching code errors or style violations,
but they are also a great source of information for learning to write idiomatic
code.</p>
<p>This article is not about linters (that a topic for another day), however,
integrating linting and testing to your git workflow allows you to catch linter
errors sooner rather than later.</p>
<p>Git has a feature called
<a rel="external" href="https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks">hooks</a> that allows a
developer to do actions based on different events. Those events could be
creating a new commit, pulling or pushing some changes, etc...</p>
<p>For example, to run your lint suite before each commit:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">cat</span><span style="color: #81C8BE;"> <<</span><span style="color: #A6D189;"> EOF</span><span style="color: #81C8BE;"> ></span><span style="color: #A6D189;"> .git/hooks/pre-commit</span></span>
<span class="giallo-l"><span style="color: #A6D189;">#!/usr/bin/env bash</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #A6D189;">exec make lint</span></span>
<span class="giallo-l"><span style="color: #A6D189;">EOF</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">chmod</span><span style="color: #A6D189;"> +x .git/hooks/pre-commit</span></span></code></pre>
<p>From now on, from now on, your lint suite run each time you create a new
commit.</p>
<p>There are some cases when you don't want to run the lint suite. This can be if
you know it will fail, or only want to create a temporary commit. You can skip
the hook by adding <code>-n</code> to <code>git commit</code>:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># Skip hook and commit in verbose mode:</span></span>
<span class="giallo-l"><span style="color: #8CAAEE;font-style: italic;">git</span><span style="color: #A6D189;"> commit -v -n</span></span></code></pre>
<p>If your test suite is fast enough, you can also add it to a git pre-commit hook.</p>
<h2 id="you-don-t-need-to-type-that-much">You don't need to type that much</h2>
<p>You might have noticed, all those pieces of advice are here to slow you down
and try to give your <a rel="external" href="https://en.wikipedia.org/wiki/Dual_process_theory">system
2</a> more chances to engage.
Most mistakes happen because we are overconfidently only relying on our system 1.
However <a rel="external" href="https://cacm.acm.org/magazines/2021/8/254304-what-does-saying-that-programming-is-hard-really-say-and-about-whom/fulltext">programming is
hard</a>,
and your System 1 has not evolved for to solve hard problems.</p>
<p>That said, you need to slow down your mind, but not necessarily your fingers.
And to avoid that and help build these habits I suggest you create bash aliases
for all those git commands. Here is an excerpt of my shell configuration:</p>
<pre class="giallo" style="color: #C6D0F5; background-color: #303446;"><code data-lang="shellscript"><span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># ~/.bashrc</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;">alias</span><span> gs</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">"git status"</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;">alias</span><span> gd</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">"git diff"</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;">alias</span><span> gds</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">"git diff --staged"</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;">alias</span><span> ga</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">"git add --patch"</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;">alias</span><span> gc</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">"git commit --verbose"</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #CA9EE6;">alias</span><span> gp</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">"git push"</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;">alias</span><span> gpf</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">"git push --force-with-lease"</span></span>
<span class="giallo-l"></span>
<span class="giallo-l"><span style="color: #949CBB;font-style: italic;"># Push a new branch:</span></span>
<span class="giallo-l"><span style="color: #CA9EE6;">alias</span><span> gpu</span><span style="color: #81C8BE;">=</span><span style="color: #A6D189;">"git push --set-upstream origin </span><span style="color: #F4B8E4;">\`</span><span style="color: #A6D189;">git rev-parse --abbrev-ref HEAD</span><span style="color: #F4B8E4;">\`</span><span style="color: #A6D189;">"</span></span></code></pre>
<p>Note the use of long-form parameters in scripts, this is a great way to
self-document each parameter.</p>
<h2 id="conclusion">Conclusion</h2>
<p>This was part one of my serie of highly effective and practical tips to become
a better engineer. I have many more areas of engineering to cover, so stay
tuned for more tips on software engineering.</p>