X1-ComputationalComplexity.tex (16127B)
1 \documentclass[11pt]{beamer} 2 \usetheme{Madrid} 3 \usepackage[utf8]{inputenc} 4 \usepackage{amsmath} 5 6 \usepackage{color} 7 \usepackage{listings} 8 \usepackage{mathtools} 9 \usepackage{tikz-cd} 10 \usepackage{adjustbox} 11 12 \definecolor{myblue}{rgb}{0,0,0.5} 13 \lstset{ 14 language=Python, 15 tabsize=4, 16 basicstyle=\footnotesize, 17 keywordstyle=\bf\color{myblue}, 18 commentstyle=\it\color{gray}, 19 numbers=left, 20 numbersep=3pt, 21 numberstyle=\tiny\color{gray}, 22 } 23 24 \author[\texttt{sebastiano.tronto@uni.lu}]{Sebastiano Tronto} 25 \title[Computational Complexity]% 26 {Why is my code slow?} 27 \logo{\includegraphics[scale=0.1]{img/unilu.jpg}} 28 %\institute{University of Luxembourg} 29 30 \date{2021-05-21} 31 32 \begin{document} 33 34 \begin{frame} 35 \titlepage 36 \end{frame} 37 38 \begin{frame}{Computational Complexity} 39 \begin{itemize} 40 \item \textbf{Goal:} 41 estimate the running {\color{blue}time} of a program 42 \item \textbf{How:} 43 count the {\color{blue}basic steps} that an 44 {\color{blue}algorithm} takes to complete 45 \item \textbf{Why}: 46 find the \emph{bottleneck} of your program, make it faster 47 \end{itemize} 48 49 \vspace{0.5cm} 50 Our analysis should not depend on the hardware 51 \end{frame} 52 53 \begin{frame}{Algorithm} 54 \begin{definition} 55 \emph{An algorithm is a sequence of {\color{blue}steps} needed to 56 solve a {\color{blue}class of problems}. } 57 \end{definition} 58 59 \begin{definition}[alternative] 60 \emph{An algorithm is a sequence of steps that takes 61 an input satisfying certain conditions and produces an output 62 satisfying other conditions.} 63 \end{definition} 64 \end{frame} 65 66 \begin{frame}{Sorting a list} 67 \begin{block}{Class of problems} 68 Sort a list $L$ of numbers in increasing order. 69 \end{block} 70 71 \begin{block}{Algorithm} 72 \begin{enumerate} 73 \item Let $S$ be an empty list. 74 \item Take an element from $L$ an insert it in $S$ in its correct 75 position. 76 \item Repeat step $2$ until $L$ is empty. 77 \item Return $S$. 78 \end{enumerate} 79 \end{block} 80 \end{frame} 81 82 \begin{frame}{Sorting a list} 83 \begin{itemize} 84 \item It solves a \emph{class} of problems: works for any list 85 \item The specific steps to sort the list $[3,7,1]$ are not an algorithm 86 \item Input conditions: must be a list of numbers 87 \item Output conditions: same numbers in increasing order 88 \end{itemize} 89 \end{frame} 90 91 \begin{frame}{How to write an algorithm} 92 \begin{itemize} 93 \item \textbf{Human language}: 94 \begin{itemize} 95 \item Easy to understand 96 \item Not precise 97 \end{itemize} 98 99 \vspace{0.3cm} 100 \item \textbf{Computer code}: 101 \begin{itemize} 102 \item Can be executed by computers 103 \item Precise 104 \item From very low level (machine code) to high level 105 (Python, \dots) 106 \end{itemize} 107 \end{itemize} 108 109 %\vspace{0.5cm} 110 %To what \emph{level of detail}? 111 \end{frame} 112 113 \begin{frame}{Basic steps} 114 \begin{itemize} 115 %\item Strictly speaking, only CPU instructions are \emph{basic} 116 %\item In practice:%, we consider basic: 117 % \begin{itemize} 118 \item Arithmetic operations $+,-,*,//,\%$ 119 \item Relational operations $==, !=, >, <,\dots$ 120 \item Memory access (read/write variable) 121 % \end{itemize} 122 \end{itemize} 123 124 \vspace{0.5cm} 125 \textbf{Warning:} 126 Depends on data type (integer, floating point, string,\dots) 127 %\begin{itemize} 128 % \item Depends on data type (integer, floating point, string,\dots) 129 % \item There are non-basic instructions such as \texttt{sort()} 130 %\end{itemize} 131 \end{frame} 132 133 \begin{frame}{Running time} 134 \begin{itemize} 135 \item Depends on computer power, programming language, compiler\dots 136 %\item Not all basic steps are equal 137 \item ``Big O'' notation: an algorithm runs in time $O(f(n))$ if, when 138 run with input of size $n$, it takes about $c\cdot f(n)$ steps 139 \item Algorithm A is \emph{asymptotically faster} than algorithm B if 140 it is faster \textbf{for $n$ large enough} 141 \item Rule of thumb: $10^7\sim10^9$ basic steps per second 142 \end{itemize} 143 \end{frame} 144 145 \begin{frame}{Asymptotical analysis vs constant factors} 146 \includegraphics[scale=0.7]{img/plot1.png} 147 \end{frame} 148 149 \begin{frame}{Asymptotical analysis vs constant factors} 150 \includegraphics[scale=0.7]{img/plot2.png} 151 \end{frame} 152 153 \begin{frame}{Asymptotical analysis vs constant factors} 154 \includegraphics[scale=0.7]{img/plot3.png} 155 \end{frame} 156 157 \begin{frame}{Asymptotical analysis vs constant factors} 158 \includegraphics[scale=0.7]{img/plot4.png} 159 \end{frame} 160 161 \begin{frame}{Asymptotical analysis vs constant factors} 162 \includegraphics[scale=0.7]{img/plot5.png} 163 \end{frame} 164 165 %\begin{frame}{title} 166 %graphs here, uncomment 167 %\end{frame} 168 169 \begin{frame}{Basic complexity analysis} 170 171 Easy things to do: 172 173 \vspace{0.3cm} 174 \begin{itemize} 175 \item Check documentation for ``non-basic steps'' 176 \begin{itemize} 177 \item Example: check Sage's \href{https://doc.sagemath.org/html/en/reference/rings\_standard/sage/rings/integer.html\#sage.rings.integer.Integer.is\_prime}{\texttt{is\_prime()}} (redirects to PARI \href{https://pari.math.u-bordeaux.fr/dochtml/html/Arithmetic\_functions.html\#se:isprime}{\texttt{isprime()}}) 178 \end{itemize} 179 180 \vspace{0.3cm} 181 \item Count nested loops 182 \begin{itemize} 183 \item How many times is a step repeated? 184 \end{itemize} 185 \end{itemize} 186 \end{frame} 187 188 {\setbeamertemplate{logo}{} 189 \begin{frame}[fragile]{Nested loops - matrix sum and product} 190 \begin{lstlisting} 191 def add(A, B): 192 n = len(A) 193 S = [[0] * n for i in range(n)] 194 for i in range(0, n): 195 for j in range(0, n): 196 S[i][j] = A[i][j] + B[i][j] 197 return S 198 \end{lstlisting} 199 200 \vspace{0.5cm} 201 \begin{lstlisting} 202 def prod(A, B): 203 n = len(A) 204 S = [[0] * n for i in range(n)] 205 for i in range(0, n): 206 for j in range(0, n): 207 for k in range(0, n): 208 S[i][j] = S[i][j] + A[i][k]*B[k][j] 209 return S 210 \end{lstlisting} 211 \end{frame} 212 } 213 214 \begin{frame}{Nested loops - matrix sum and product} 215 \begin{itemize} 216 \item \texttt{add} is $O(n^2)$ (two loops) 217 \item \texttt{prod} is $O(n^3)$ (three loops) 218 \end{itemize} 219 220 \vspace{0.3cm} 221 \textbf{Fun fact:} there are faster algorithms for matrix multiplication, 222 for example \href{https://en.wikipedia.org/wiki/Strassen_algorithm}% 223 {Strassen's algorithm}. 224 \end{frame} 225 226 \begin{frame}[fragile]{Sorting a list} 227 \begin{lstlisting} 228 def correct_position(e, S): 229 for i in range(0, len(S)): 230 if S[i] > e: 231 return i 232 return len(S) 233 234 def sort_list(L): 235 S = [] 236 for e in L: 237 cp = correct_position(e, S) 238 S.insert(cp, e) 239 return S 240 \end{lstlisting} 241 \end{frame} 242 243 \begin{frame}{Sorting a list} 244 \begin{itemize} 245 \item Complexity of \texttt{correct\_position()}: 246 \begin{itemize} 247 %\item best case $O(1)$ 248 \item worst case $O($\texttt{len(S)}$)$ 249 \item average $O($\texttt{len(S)}$)$ 250 \end{itemize} 251 252 \vspace{0.3cm} 253 \item Complexity of \texttt{sort\_list} (here $n=$\texttt{len(L)}): 254 \begin{align*} 255 %\sum_{i=0}^{n-1} O(1) = O(n) && \text{best case}\\ 256 \sum_{i=0}^{n-1} O(i) = O(n^2)% && \text{average/worst} 257 \end{align*} 258 (it calls \texttt{correct\_position()} $n$ times). 259 \end{itemize} 260 \end{frame} 261 262 \begin{frame}{Sorting a list} 263 \begin{itemize} 264 \item For which lists does the ``best case'' happen? 265 \item For which lists does the ``worst case'' happen? 266 \item How large can $n$ be for \texttt{sort\_list()} to run 267 in under a second? 268 \end{itemize} 269 \end{frame} 270 271 \begin{frame}{Sorting a list} 272 How to improve our code? 273 \begin{itemize} 274 \item Improve \texttt{correct\_position()} 275 \item Take advantage of the fact that $S$ is always sorted 276 \end{itemize} 277 \end{frame} 278 279 \begin{frame}{Binary search} 280 \begin{block}{Algorithm} 281 \textbf{Input:} a \emph{sorted} list $S$ and a value $e$. 282 \begin{enumerate} 283 \item If the list is empty, you have found the position of $e$ 284 \item Otherwise, compare $e$ to the middle element $m$ of $S$ 285 \begin{itemize} 286 \item If $e<m$, repeat from (1) on the first half of $S$ 287 \item Otherwise, repeat from (1) on the second half of $S$ 288 \end{itemize} 289 \end{enumerate} 290 \end{block} 291 \end{frame} 292 293 \begin{frame}[fragile]{Binary search} 294 \begin{lstlisting} 295 # Return position of e in L 296 def binary_search(e, S, start, end): 297 if start == end: 298 return start 299 midpoint = (end+start)//2 300 if e < S[midpoint]: 301 return binary_search(e, S, start, midpoint) 302 else: 303 return binary_search(e, S, midpoint+1, end) 304 \end{lstlisting} 305 \end{frame} 306 307 \begin{frame}{Binary search - example 1} 308 Searching for \texttt{e}$=2$: 309 \begin{align*} 310 \only<1>{ 311 \underbrace{ 312 \overset{{\color{blue} 313 \substack{\mathclap{\texttt{start}=0}\\\downarrow}}}{-2} 314 \quad 0\quad 1\quad 3\quad 315 \overset{\substack{\mathclap{\texttt{midpoint}=4}\\\downarrow}}{5} 316 \quad 6\quad 7\quad 9\quad 12 317 }\quad 318 \overset{{\color{red} 319 \substack{\mathclap{\texttt{end}=9}\\\downarrow}}}{\phantom{0}} 320 } 321 \only<2>{ 322 \underbrace{ 323 \overset{{\color{blue} 324 \substack{\mathclap{\texttt{start}=0}\\\downarrow}}}{-2} 325 \quad 0\quad 326 \overset{\substack{\mathclap{\texttt{midpoint}=2}\\\\\downarrow}}% 327 {1} 328 \quad 3 329 }\quad 330 \overset{{\color{red} 331 \substack{\mathclap{\texttt{end}=4}\\\downarrow}}}{5} 332 \quad 6\quad 7\quad 9\quad 12\quad \phantom{0} 333 } 334 \only<3>{ 335 -2\quad 0\quad 1\quad 336 \underbrace{ 337 \overset{ 338 \substack{ 339 \mathclap{ 340 {\color{blue}\texttt{start}}=\texttt{midpoint}=3}\\\\ 341 {\color{blue}\downarrow} 342 } 343 }{3} 344 } \quad 345 \overset{{\color{red} 346 \substack{\mathclap{\texttt{end}=4}\\\downarrow}}}{5} 347 \quad 6\quad 7\quad 9\quad 12\quad \phantom{0} 348 } 349 \only<4>{ 350 -2\quad 0\quad 1\quad 351 \overset{ 352 \substack{ 353 \mathclap{ 354 {\color{blue}\texttt{start}}= 355 {\color{red}\texttt{end}}=3}\\\downarrow}}{3} 356 \quad 5 \quad 6\quad 7\quad 9\quad 12\quad \phantom{0} 357 } 358 \end{align*} 359 \only<1>{{\color{blue}$e<5$}$\implies$ check left half} 360 \only<2>{{\color{red}$e>1$}$\implies$ check right half} 361 \only<3>{{\color{blue}$e<3$}$\implies$ check left half} 362 \only<4>{\texttt{start}=\texttt{end}, done} 363 \end{frame} 364 365 \begin{frame}{Binary search - example 2} 366 Searching for \texttt{e}$=11$: 367 \begin{align*} 368 \only<1>{ 369 \underbrace{ 370 \overset{{\color{blue} 371 \substack{\mathclap{\texttt{start}=0}\\\downarrow}}}{-2} 372 \quad 0\quad 1\quad 3\quad 373 \overset{\substack{\mathclap{\texttt{midpoint}=4}\\\downarrow}}{5} 374 \quad 6\quad 7\quad 9\quad 12 375 }\quad 376 \overset{{\color{red} 377 \substack{\mathclap{\texttt{end}=9}\\\downarrow}}}{\phantom{0}} 378 } 379 \only<2>{ 380 -2 \quad 0\quad 1 \quad 3 \quad 5 \quad 381 \underbrace{ 382 \overset{{\color{blue} 383 \substack{\mathclap{\texttt{start}=5}\\\downarrow}}}{6} 384 \quad 7 \quad 385 \overset{\substack{\mathclap{\texttt{midpoint}=7}\\\\\downarrow}}% 386 {9} 387 \quad 12 388 }\quad 389 \overset{{\color{red} 390 \substack{\mathclap{\texttt{end}=9}\\\downarrow}}}{\phantom{0}} 391 } 392 \only<3>{ 393 -2\quad 0\quad 1\quad 3\quad 5\quad 6\quad 7\quad 9\quad 394 \underbrace{ 395 \overset{ 396 \substack{ 397 \mathclap{ 398 {\color{blue}\texttt{start}}=\texttt{midpoint}=8}\\\\ 399 {\color{blue}\downarrow} 400 } 401 }{12} 402 } \quad 403 \overset{{\color{red} 404 \substack{\mathclap{\texttt{end}=9}\\\downarrow}}}{\phantom{0}} 405 } 406 \only<4>{ 407 -2\quad 0\quad 1\quad 3\quad 5\quad 6\quad 7\quad 9\quad 408 \overset{ 409 \substack{ 410 \mathclap{ 411 {\color{blue}\texttt{start}}= 412 {\color{red}\texttt{end}}=8}\\\downarrow}}{12} 413 } 414 \end{align*} 415 \only<1>{{\color{red}$e>5$}$\implies$ check right half} 416 \only<2>{{\color{red}$e>9$}$\implies$ check right half} 417 \only<3>{{\color{blue}$e<11$}$\implies$ check left half} 418 \only<4>{\texttt{start}=\texttt{end}, done} 419 \end{frame} 420 421 \begin{frame}{Binary search} 422 \begin{itemize} 423 \item Works only if the list is sorted 424 \item Complexity $O(\log_2(n))$: at every step we cut the list in half 425 \item Recursive, \emph{divide et impera} 426 \end{itemize} 427 \end{frame} 428 429 \begin{frame}[fragile]{Sorting a list - binary search version} 430 \begin{lstlisting} 431 def sort_list(L): 432 S = [] 433 for e in L: 434 cp = binary_search(e, S, 0, len(S)) # This changed 435 S.insert(cp, e) 436 return S 437 \end{lstlisting} 438 \vspace{0.3cm} 439 \begin{itemize} 440 \item Complexity: \[\sum_{i=0}^{n-1} O(\log_2(i)) = O(n\log_2(n))\]\\ 441 (it calls \texttt{binary\_search} $n$ times). 442 \end{itemize} 443 \end{frame} 444 445 \begin{frame}{Fast exponentiation} 446 \begin{block}{Algorithm / formula} 447 \begin{align*} 448 a^n= 449 \begin{cases} 450 1 & \text{if }n=0,\\ 451 (a\cdot a)^{\frac n2} & \text{if $n$ is even},\\ 452 a\cdot a^{n-1} & \text{if $n$ is odd.} 453 \end{cases} 454 \end{align*} 455 \end{block} 456 \end{frame} 457 458 \begin{frame}[fragile]{Fast exponentiation} 459 \begin{lstlisting} 460 # Compute a^n (n>=0 integer) 461 def power(a, n): 462 if n == 0: 463 return 1 464 if n % 2 == 0: # n is even 465 return power(a*a, n//2) 466 else: # n is odd 467 return a*power(a, n-1) 468 \end{lstlisting} 469 \end{frame} 470 471 \begin{frame}{Fast exponentiation} 472 473 \begin{itemize} 474 \item Complexity: $O(\log_2(n))$ (after $2$ steps, $n$ is halved) 475 \item Python's operator $**$ does something similar 476 \item Naive algorithm (one loop): $O(n)$ 477 \end{itemize} 478 \end{frame} 479 480 481 \begin{frame}[fragile]{Fast $\gcd$} 482 \begin{block}{Algorithm / formula} 483 \begin{align*} 484 \gcd(a,b) = 485 \begin{cases} 486 a & \text{if }b=0,\\ 487 \gcd(b,a\bmod b) & \text{otherwise.} 488 \end{cases} 489 \end{align*} 490 \end{block} 491 492 \begin{columns} 493 \column{0.5\textwidth} 494 \begin{lstlisting} 495 def gcd(a, b): 496 if b == 0: 497 return a 498 else: 499 return gcd(b, a%b) 500 \end{lstlisting} 501 502 \column{0.5\textwidth} 503 \begin{itemize} 504 \item After $2$ steps, $a$ is halved $\implies$ complexity $O(\log_2(a))$ 505 \end{itemize} 506 \end{columns} 507 \end{frame} 508 509 \begin{frame}{Recursion} 510 \begin{itemize} 511 \item These examples use \emph{recursion} 512 (a function that calls itself) 513 \item If it calls itself more than once, it is slow 514 (\emph{exponential} complexity!) 515 \end{itemize} 516 \end{frame} 517 518 \begin{frame}[fragile]{Fibonacci numbers} 519 520 \begin{block}{Algorithm / formula} 521 \begin{align*} 522 F(n) = 523 \begin{cases} 524 n & \text{if }n\leq1,\\ 525 F(n-1)+F(n-2) & \text{otherwise.} 526 \end{cases} 527 \end{align*} 528 \end{block} 529 530 \vspace{0.5cm} 531 \begin{lstlisting} 532 def F(n): 533 if n <= 1: 534 return n 535 else: 536 return F(n-1) + F(n-2) 537 \end{lstlisting} 538 \end{frame} 539 540 \begin{frame}[fragile]{Fibonacci} 541 \begin{adjustbox}{scale={0.85}{0.9},center} 542 \begin{tikzcd}[column sep=1mm] 543 & & & & & & & & F(5) \ar[drrr] \ar[dlll]\\ 544 & & & & & F(4)\ar[dll]\ar[dr] & & & & & & F(3) \ar[dl] \ar[dr]\\ 545 & & & F(3) \ar[dl]\ar[dr] & & & F(2) \ar[dr]\ar[dl] 546 & & & & F(2) \ar[dl]\ar[dr] & & F(1) \\ 547 & & F(2) \ar[dl]\ar[dr] & & F(1) & F(1) & & F(0) & & F(1) & & F(0)\\ 548 & F(1) & & F(0) 549 \end{tikzcd} 550 \end{adjustbox} 551 \end{frame} 552 553 \begin{frame}{Fibonacci} 554 \begin{itemize} 555 \item Complexity: almost $O(2^n)$ (actually $O(\varphi^n)$ 556 with $\varphi=\frac{1+\sqrt 5}{2}\sim 1.6$) 557 \item But some values are computed many times! 558 \item Optimization: memorize previously computed values 559 \end{itemize} 560 \end{frame} 561 562 \begin{frame}[fragile]{Fibonacci with memorization} 563 \begin{lstlisting} 564 # List with memorized values, N is the largest possible 565 N = 10**6 566 F_memorized = [-1] * N 567 568 def F(n): 569 if F_memorized[n] == -1: 570 if n <= 1: 571 F_memorized[n] = n 572 else: 573 F_memorized[n] = F(n-1) + F(n-2) 574 575 return F_memorized[n] 576 \end{lstlisting} 577 \end{frame} 578 579 \begin{frame}[fragile]{Fibonacci with memorization} 580 \begin{adjustbox}{scale={0.85}{0.9},center} 581 \begin{tikzcd}[column sep=1mm] 582 & & & & & & & & F(5) \ar[drrr] \ar[dlll]\\ 583 & & & & & F(4)\ar[dll]\ar[dr] & & & & & & {\color{blue}F(3)}\\ 584 & & & F(3) \ar[dl]\ar[dr] & & & {\color{blue}F(2)}\\ 585 & & F(2) \ar[dl]\ar[dr] & & {\color{blue}F(1)} \\ 586 & F(1) & & F(0) 587 \end{tikzcd} 588 \end{adjustbox} 589 \end{frame} 590 591 \begin{frame}{Fibonacci with memorization} 592 \begin{itemize} 593 \item Complexity: $O(n)$, huge improvement! 594 \item Further improvement (but still $O(n)$): dynamic programming 595 \item Pay attention to memory usage 596 \end{itemize} 597 \end{frame} 598 599 \begin{frame}{References} 600 \begin{itemize} 601 \item Thomas H. Cormen, Charles E. Leiserson, Ronald L. Rivest, and 602 Clifford Stein - 603 \href{https://en.wikipedia.org/wiki/Introduction\_to\_Algorithms}% 604 {\emph{Introductions to Algorithms}} 605 \end{itemize} 606 \end{frame} 607 608 \end{document}